mpidpkt.h 36.6 KB
Newer Older
1
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
2 3 4 5 6 7 8
/*
 *  (C) 2001 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

#ifndef HAVE_MPIDPKT_H
#define HAVE_MPIDPKT_H
9

10 11
#include "oputil.h"

12 13 14 15 16 17 18
#ifdef HAVE_STDINT_H
#  include <stdint.h>
#endif
#ifdef HAVE_INTTYPES_H
#  include <inttypes.h>
#endif

19 20 21
/* Enable the use of data within the message packet for small messages */
#define USE_EAGER_SHORT
#define MPIDI_EAGER_SHORT_INTS 4
22
/* FIXME: This appears to assume that sizeof(int) == 4 (or at least >= 4) */
23 24
#define MPIDI_EAGER_SHORT_SIZE 16

25
/* This is the number of ints that can be carried within an RMA packet */
Xin Zhao's avatar
Xin Zhao committed
26
#define MPIDI_RMA_IMMED_BYTES 16
27

28 29 30
/* Union over all types (integer, logical, and multi-language types) that are
   allowed in a CAS operation.  This is used to allocate enough space in the
   packet header for immediate data.  */
31
typedef union {
32
#define MPIR_OP_TYPE_MACRO(mpi_type_, c_type_, type_name_) c_type_ cas_##type_name_;
33 34 35 36 37 38 39 40
    MPIR_OP_TYPE_GROUP(C_INTEGER)
    MPIR_OP_TYPE_GROUP(FORTRAN_INTEGER)
    MPIR_OP_TYPE_GROUP(LOGICAL)
    MPIR_OP_TYPE_GROUP(BYTE)
    MPIR_OP_TYPE_GROUP(C_INTEGER_EXTRA)
    MPIR_OP_TYPE_GROUP(FORTRAN_INTEGER_EXTRA)
    MPIR_OP_TYPE_GROUP(LOGICAL_EXTRA)
    MPIR_OP_TYPE_GROUP(BYTE_EXTRA)
41
#undef MPIR_OP_TYPE_MACRO
42 43
} MPIDI_CH3_CAS_Immed_u;

44 45 46 47
/* Union over all types (all predefined types) that are allowed in a
   Fetch-and-op operation.  This can be too large for the packet header, so we
   limit the immediate space in the header to FOP_IMMED_INTS. */

48 49 50
/* *INDENT-OFF* */
/* Indentation turned off because "indent" is getting confused with
 * the lack of a semi-colon in the fields below */
51
typedef union {
52
#define MPIR_OP_TYPE_MACRO(mpi_type_, c_type_, type_name_) c_type_ fop##type_name_;
53 54
    MPIR_OP_TYPE_GROUP_ALL_BASIC
    MPIR_OP_TYPE_GROUP_ALL_EXTRA
55
#undef MPIR_OP_TYPE_MACRO
56
} MPIDI_CH3_FOP_Immed_u;
57
/* *INDENT-ON* */
58

59
/*
60
 * Predefined packet types.  This simplifies some of the code.
61
 */
62 63
/* FIXME: Having predefined names makes it harder to add new message types,
   such as different RMA types. */
64
typedef enum {
Xin Zhao's avatar
Xin Zhao committed
65
    MPIDI_CH3_PKT_EAGER_SEND = 0,
66
#if defined(USE_EAGER_SHORT)
67
    MPIDI_CH3_PKT_EAGERSHORT_SEND,
68
#endif /* defined(USE_EAGER_SHORT) */
69
    MPIDI_CH3_PKT_EAGER_SYNC_SEND,      /* FIXME: no sync eager */
70 71 72 73
    MPIDI_CH3_PKT_EAGER_SYNC_ACK,
    MPIDI_CH3_PKT_READY_SEND,
    MPIDI_CH3_PKT_RNDV_REQ_TO_SEND,
    MPIDI_CH3_PKT_RNDV_CLR_TO_SEND,
74
    MPIDI_CH3_PKT_RNDV_SEND,    /* FIXME: should be stream put */
75 76
    MPIDI_CH3_PKT_CANCEL_SEND_REQ,
    MPIDI_CH3_PKT_CANCEL_SEND_RESP,
Xin Zhao's avatar
Xin Zhao committed
77 78
    /* RMA Packets begin here */
    MPIDI_CH3_PKT_PUT,
79 80
    MPIDI_CH3_PKT_GET,
    MPIDI_CH3_PKT_ACCUMULATE,
Xin Zhao's avatar
Xin Zhao committed
81 82 83 84 85 86 87
    MPIDI_CH3_PKT_GET_ACCUM,
    MPIDI_CH3_PKT_FOP,
    MPIDI_CH3_PKT_CAS,
    MPIDI_CH3_PKT_GET_RESP,
    MPIDI_CH3_PKT_GET_ACCUM_RESP,
    MPIDI_CH3_PKT_FOP_RESP,
    MPIDI_CH3_PKT_CAS_RESP,
88
    MPIDI_CH3_PKT_LOCK,
89
    MPIDI_CH3_PKT_LOCK_GRANTED,
90
    MPIDI_CH3_PKT_UNLOCK,
91
    MPIDI_CH3_PKT_FLUSH,
92
    MPIDI_CH3_PKT_FLUSH_ACK,
93
    MPIDI_CH3_PKT_DECR_AT_COUNTER,
94 95
    /* RMA Packets end here */
    MPIDI_CH3_PKT_FLOW_CNTL_UPDATE,     /* FIXME: Unused */
96
    MPIDI_CH3_PKT_CLOSE,
Wesley Bland's avatar
Wesley Bland committed
97
    MPIDI_CH3_PKT_REVOKE,
Pavan Balaji's avatar
Pavan Balaji committed
98
    MPIDI_CH3_PKT_END_CH3,
99
    /* The channel can define additional types by defining the value
100
     * MPIDI_CH3_PKT_ENUM */
101
# if defined(MPIDI_CH3_PKT_ENUM)
Pavan Balaji's avatar
Pavan Balaji committed
102
    MPIDI_CH3_PKT_ENUM,
103
# endif
Pavan Balaji's avatar
Pavan Balaji committed
104
    MPIDI_CH3_PKT_END_ALL,
105
    MPIDI_CH3_PKT_INVALID = -1  /* forces a signed enum to quash warnings */
Pavan Balaji's avatar
Pavan Balaji committed
106
} MPIDI_CH3_Pkt_type_t;
107

108
/* These flags can be "OR'ed" together */
109 110 111 112 113 114
typedef enum {
    MPIDI_CH3_PKT_FLAG_NONE = 0,
    MPIDI_CH3_PKT_FLAG_RMA_LOCK = 1,
    MPIDI_CH3_PKT_FLAG_RMA_UNLOCK = 2,
    MPIDI_CH3_PKT_FLAG_RMA_FLUSH = 4,
    MPIDI_CH3_PKT_FLAG_RMA_REQ_ACK = 8,
115
    MPIDI_CH3_PKT_FLAG_RMA_DECR_AT_COUNTER = 16,
116 117
    MPIDI_CH3_PKT_FLAG_RMA_NOCHECK = 32,
    MPIDI_CH3_PKT_FLAG_RMA_SHARED = 64,
118
    MPIDI_CH3_PKT_FLAG_RMA_EXCLUSIVE = 128,
119
    MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK = 256,
120 121 122 123 124 125
    MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED = 512,
    MPIDI_CH3_PKT_FLAG_RMA_LOCK_QUEUED_DATA_QUEUED = 1024,
    MPIDI_CH3_PKT_FLAG_RMA_LOCK_QUEUED_DATA_DISCARDED = 2048,
    MPIDI_CH3_PKT_FLAG_RMA_LOCK_DISCARDED = 4096,
    MPIDI_CH3_PKT_FLAG_RMA_UNLOCK_NO_ACK = 8192,
    MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP = 16384
126
} MPIDI_CH3_Pkt_flags_t;
127

128
typedef struct MPIDI_CH3_Pkt_send {
129 130
    MPIDI_CH3_Pkt_type_t type;  /* XXX - uint8_t to conserve space ??? */
    MPIDI_Message_match match;
131
    MPI_Request sender_req_id;  /* needed for ssend and send cancel */
132 133 134
    MPIDI_msg_sz_t data_sz;
#if defined(MPID_USE_SEQUENCE_NUMBERS)
    MPID_Seqnum_t seqnum;
135 136
#endif
} MPIDI_CH3_Pkt_send_t;
137

138
/* NOTE: Normal and synchronous eager sends, as well as all ready-mode sends,
139 140 141 142
   use the same structure but have a different type value. */
typedef MPIDI_CH3_Pkt_send_t MPIDI_CH3_Pkt_eager_send_t;
typedef MPIDI_CH3_Pkt_send_t MPIDI_CH3_Pkt_eager_sync_send_t;
typedef MPIDI_CH3_Pkt_send_t MPIDI_CH3_Pkt_ready_send_t;
143 144

#if defined(USE_EAGER_SHORT)
145
typedef struct MPIDI_CH3_Pkt_eagershort_send {
146 147 148 149 150 151
    MPIDI_CH3_Pkt_type_t type;  /* XXX - uint8_t to conserve space ??? */
    MPIDI_Message_match match;
    MPIDI_msg_sz_t data_sz;
#if defined(MPID_USE_SEQUENCE_NUMBERS)
    MPID_Seqnum_t seqnum;
#endif
152 153
    int data[MPIDI_EAGER_SHORT_INTS];   /* FIXME: Experimental for now */
} MPIDI_CH3_Pkt_eagershort_send_t;
154
#endif /* defined(USE_EAGER_SHORT) */
155

156
typedef struct MPIDI_CH3_Pkt_eager_sync_ack {
157 158
    MPIDI_CH3_Pkt_type_t type;
    MPI_Request sender_req_id;
159
} MPIDI_CH3_Pkt_eager_sync_ack_t;
160 161 162

typedef MPIDI_CH3_Pkt_send_t MPIDI_CH3_Pkt_rndv_req_to_send_t;

163
typedef struct MPIDI_CH3_Pkt_rndv_clr_to_send {
164 165 166
    MPIDI_CH3_Pkt_type_t type;
    MPI_Request sender_req_id;
    MPI_Request receiver_req_id;
167
} MPIDI_CH3_Pkt_rndv_clr_to_send_t;
168

169
typedef struct MPIDI_CH3_Pkt_rndv_send {
170 171
    MPIDI_CH3_Pkt_type_t type;
    MPI_Request receiver_req_id;
172
} MPIDI_CH3_Pkt_rndv_send_t;
173

174
typedef struct MPIDI_CH3_Pkt_cancel_send_req {
175 176 177
    MPIDI_CH3_Pkt_type_t type;
    MPIDI_Message_match match;
    MPI_Request sender_req_id;
178
} MPIDI_CH3_Pkt_cancel_send_req_t;
179

180
typedef struct MPIDI_CH3_Pkt_cancel_send_resp {
181 182 183
    MPIDI_CH3_Pkt_type_t type;
    MPI_Request sender_req_id;
    int ack;
184
} MPIDI_CH3_Pkt_cancel_send_resp_t;
185

186 187 188
/* *INDENT-OFF* */
/* Indentation turned off because "indent" is getting confused with
 * the lack of a semi-colon in the field below */
189 190 191
#if defined(MPIDI_CH3_PKT_DEFS)
MPIDI_CH3_PKT_DEFS
#endif
192
/* *INDENT-ON* */
193

194 195 196
#define MPIDI_CH3_PKT_RMA_GET_TARGET_DATATYPE(pkt_, datatype_, err_)    \
    {                                                                   \
        err_ = MPI_SUCCESS;                                             \
Xin Zhao's avatar
Xin Zhao committed
197
        switch((pkt_).type) {                                           \
198
        case (MPIDI_CH3_PKT_PUT):                                       \
Xin Zhao's avatar
Xin Zhao committed
199
            datatype_ = (pkt_).put.datatype;                            \
200 201
            break;                                                      \
        case (MPIDI_CH3_PKT_GET):                                       \
Xin Zhao's avatar
Xin Zhao committed
202
            datatype_ = (pkt_).get.datatype;                            \
203 204
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
Xin Zhao's avatar
Xin Zhao committed
205
            datatype_ = (pkt_).accum.datatype;                          \
206
            break;                                                      \
Xin Zhao's avatar
Xin Zhao committed
207
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
Xin Zhao's avatar
Xin Zhao committed
208
            datatype_ = (pkt_).get_accum.datatype;                      \
Xin Zhao's avatar
Xin Zhao committed
209
            break;                                                      \
210
        case (MPIDI_CH3_PKT_CAS):                                       \
Xin Zhao's avatar
Xin Zhao committed
211
            datatype_ = (pkt_).cas.datatype;                            \
212 213
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP):                                       \
Xin Zhao's avatar
Xin Zhao committed
214
            datatype_ = (pkt_).fop.datatype;                            \
215 216
            break;                                                      \
        default:                                                        \
Xin Zhao's avatar
Xin Zhao committed
217
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
218 219 220
        }                                                               \
    }

221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293
#define MPIDI_CH3_PKT_RMA_GET_TARGET_COUNT(pkt_, count_, err_)          \
    {                                                                   \
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_PUT):                                       \
            count_ = (pkt_).put.count;                                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET):                                       \
            count_ = (pkt_).get.count;                                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
            count_ = (pkt_).accum.count;                                \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
            count_ = (pkt_).get_accum.count;                            \
            break;                                                      \
        case (MPIDI_CH3_PKT_CAS):                                       \
        case (MPIDI_CH3_PKT_FOP):                                       \
            count_ = 1;                                                 \
            break;                                                      \
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

#define MPIDI_CH3_PKT_RMA_GET_IMMED_LEN(pkt_, immed_len_, err_)         \
    {                                                                   \
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_PUT):                                       \
            immed_len_ = (pkt_).put.immed_len;                          \
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
            immed_len_ = (pkt_).accum.immed_len;                        \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
            immed_len_ = (pkt_).get_accum.immed_len;                    \
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP):                                       \
            immed_len_ = (pkt_).fop.immed_len;                          \
            break;                                                      \
        case (MPIDI_CH3_PKT_CAS):                                       \
            /* FIXME: we should deal with CAS here */                   \
            break;                                                      \
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

#define MPIDI_CH3_PKT_RMA_GET_IMMED_DATA_PTR(pkt_, immed_data_, err_)   \
    {                                                                   \
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_PUT):                                       \
            immed_data_ = (pkt_).put.data;                              \
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
            immed_data_ = (pkt_).accum.data;                            \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
            immed_data_ = (pkt_).get_accum.data;                        \
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP):                                       \
            immed_data_ = (pkt_).fop.data;                              \
            break;                                                      \
        case (MPIDI_CH3_PKT_CAS):                                       \
            /* FIXME: we should deal with CAS here */                   \
            break;                                                      \
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

Xin Zhao's avatar
Xin Zhao committed
294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380
#define MPIDI_CH3_PKT_RMA_GET_LOCK_TYPE(pkt_, lock_type_, err_)         \
    {                                                                   \
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_PUT):                                       \
            lock_type_ = (pkt_).put.lock_type;                          \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET):                                       \
            lock_type_ = (pkt_).get.lock_type;                          \
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
            lock_type_ = (pkt_).accum.lock_type;                        \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
            lock_type_ = (pkt_).get_accum.lock_type;                    \
            break;                                                      \
        case (MPIDI_CH3_PKT_CAS):                                       \
            lock_type_ = (pkt_).cas.lock_type;                          \
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP):                                       \
            lock_type_ = (pkt_).fop.lock_type;                          \
            break;                                                      \
        case (MPIDI_CH3_PKT_LOCK):                                      \
            lock_type_ = (pkt_).lock.lock_type;                         \
            break;                                                      \
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

#define MPIDI_CH3_PKT_RMA_GET_ORIGIN_RANK(pkt_, origin_rank_, err_)     \
    {                                                                   \
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_PUT):                                       \
            origin_rank_ = (pkt_).put.origin_rank;                      \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET):                                       \
            origin_rank_ = (pkt_).get.origin_rank;                      \
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
            origin_rank_ = (pkt_).accum.origin_rank;                    \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
            origin_rank_ = (pkt_).get_accum.origin_rank;                \
            break;                                                      \
        case (MPIDI_CH3_PKT_CAS):                                       \
            origin_rank_ = (pkt_).cas.origin_rank;                      \
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP):                                       \
            origin_rank_ = (pkt_).fop.origin_rank;                      \
            break;                                                      \
        case (MPIDI_CH3_PKT_LOCK):                                      \
            origin_rank_ = (pkt_).lock.origin_rank;                     \
            break;                                                      \
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

#define MPIDI_CH3_PKT_RMA_GET_FLAGS(pkt_, flags_, err_)                 \
    {                                                                   \
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_PUT):                                       \
            flags_ = (pkt_).put.flags;                                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET):                                       \
            flags_ = (pkt_).get.flags;                                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
            flags_ = (pkt_).accum.flags;                                \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
            flags_ = (pkt_).get_accum.flags;                            \
            break;                                                      \
        case (MPIDI_CH3_PKT_CAS):                                       \
            flags_ = (pkt_).cas.flags;                                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP):                                       \
            flags_ = (pkt_).fop.flags;                                  \
            break;                                                      \
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437
#define MPIDI_CH3_PKT_RMA_ERASE_FLAGS(pkt_, err_)                       \
    {                                                                   \
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_PUT):                                       \
            (pkt_).put.flags = MPIDI_CH3_PKT_FLAG_NONE;                 \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET):                                       \
            (pkt_).get.flags = MPIDI_CH3_PKT_FLAG_NONE;                 \
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
            (pkt_).accum.flags = MPIDI_CH3_PKT_FLAG_NONE;               \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
            (pkt_).get_accum.flags = MPIDI_CH3_PKT_FLAG_NONE;           \
            break;                                                      \
        case (MPIDI_CH3_PKT_CAS):                                       \
            (pkt_).cas.flags = MPIDI_CH3_PKT_FLAG_NONE;                 \
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP):                                       \
            (pkt_).fop.flags = MPIDI_CH3_PKT_FLAG_NONE;                 \
            break;                                                      \
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

#define MPIDI_CH3_PKT_RMA_GET_SOURCE_WIN_HANDLE(pkt_, win_hdl_, err_)   \
    {                                                                   \
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_PUT):                                       \
            win_hdl_ = (pkt_).put.source_win_handle;                    \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET):                                       \
            win_hdl_ = (pkt_).get.source_win_handle;                    \
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
            win_hdl_ = (pkt_).accum.source_win_handle;                  \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
            win_hdl_ = (pkt_).get_accum.source_win_handle;              \
            break;                                                      \
        case (MPIDI_CH3_PKT_CAS):                                       \
            win_hdl_ = (pkt_).cas.source_win_handle;                    \
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP):                                       \
            win_hdl_ = (pkt_).fop.source_win_handle;                    \
            break;                                                      \
        case (MPIDI_CH3_PKT_LOCK):                                      \
            win_hdl_ = (pkt_).lock.source_win_handle;                   \
            break;                                                      \
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

438
#define MPIDI_CH3_PKT_RMA_GET_TARGET_WIN_HANDLE(pkt_, win_hdl_, err_)   \
Xin Zhao's avatar
Xin Zhao committed
439 440 441 442
    {                                                                   \
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_PUT):                                       \
443
            win_hdl_ = (pkt_).put.target_win_handle;                    \
Xin Zhao's avatar
Xin Zhao committed
444 445
            break;                                                      \
        case (MPIDI_CH3_PKT_GET):                                       \
446
            win_hdl_ = (pkt_).get.target_win_handle;                    \
Xin Zhao's avatar
Xin Zhao committed
447 448
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
449
            win_hdl_ = (pkt_).accum.target_win_handle;                  \
Xin Zhao's avatar
Xin Zhao committed
450 451
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
452
            win_hdl_ = (pkt_).get_accum.target_win_handle;              \
Xin Zhao's avatar
Xin Zhao committed
453 454
            break;                                                      \
        case (MPIDI_CH3_PKT_CAS):                                       \
455
            win_hdl_ = (pkt_).cas.target_win_handle;                    \
Xin Zhao's avatar
Xin Zhao committed
456 457
            break;                                                      \
        case (MPIDI_CH3_PKT_FOP):                                       \
458
            win_hdl_ = (pkt_).fop.target_win_handle;                    \
Xin Zhao's avatar
Xin Zhao committed
459
            break;                                                      \
460 461 462
        case (MPIDI_CH3_PKT_LOCK):                                      \
            win_hdl_ = (pkt_).lock.target_win_handle;                   \
            break;                                                      \
Xin Zhao's avatar
Xin Zhao committed
463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }

#define MPIDI_CH3_PKT_RMA_SET_DATALOOP_SIZE(pkt_, dataloop_size_, err_) \
    {                                                                   \
        err_ = MPI_SUCCESS;                                             \
        switch((pkt_).type) {                                           \
        case (MPIDI_CH3_PKT_PUT):                                       \
            (pkt_).put.dataloop_size = (dataloop_size_);                \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET):                                       \
            (pkt_).get.dataloop_size = (dataloop_size_);                \
            break;                                                      \
        case (MPIDI_CH3_PKT_ACCUMULATE):                                \
            (pkt_).accum.dataloop_size = (dataloop_size_);              \
            break;                                                      \
        case (MPIDI_CH3_PKT_GET_ACCUM):                                 \
            (pkt_).get_accum.dataloop_size = (dataloop_size_);          \
            break;                                                      \
        default:                                                        \
            MPIU_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \
        }                                                               \
    }


490
typedef struct MPIDI_CH3_Pkt_put {
491
    MPIDI_CH3_Pkt_type_t type;
492
    MPIDI_CH3_Pkt_flags_t flags;
493 494 495
    void *addr;
    int count;
    MPI_Datatype datatype;
496 497 498 499 500 501 502 503
    int dataloop_size;          /* for derived datatypes */
    MPI_Win target_win_handle;  /* Used in the last RMA operation in each
                                 * epoch for decrementing rma op counter in
                                 * active target rma and for unlocking window
                                 * in passive target rma. Otherwise set to NULL*/
    MPI_Win source_win_handle;  /* Used in the last RMA operation in an
                                 * epoch in the case of passive target rma
                                 * with shared locks. Otherwise set to NULL*/
504
    char data[MPIDI_RMA_IMMED_BYTES];
505 506 507
    size_t immed_len;
    int lock_type;      /* used when piggybacking LOCK message. */
    int origin_rank;    /* used when piggybacking LOCK message. */
508 509 510
} MPIDI_CH3_Pkt_put_t;

typedef struct MPIDI_CH3_Pkt_get {
511
    MPIDI_CH3_Pkt_type_t type;
512
    MPIDI_CH3_Pkt_flags_t flags;
513 514 515
    void *addr;
    int count;
    MPI_Datatype datatype;
516
    int dataloop_size;          /* for derived datatypes */
517
    MPI_Request request_handle;
518 519 520 521 522 523 524 525 526
    MPI_Win target_win_handle;  /* Used in the last RMA operation in each
                                 * epoch for decrementing rma op counter in
                                 * active target rma and for unlocking window
                                 * in passive target rma. Otherwise set to NULL*/
    MPI_Win source_win_handle;  /* Used in the last RMA operation in an
                                 * epoch in the case of passive target rma
                                 * with shared locks. Otherwise set to NULL*/
    int lock_type;   /* used when piggybacking LOCK message. */
    int origin_rank; /* used when piggybacking LOCK message. */
527 528
} MPIDI_CH3_Pkt_get_t;

529 530 531 532 533 534 535 536 537
typedef struct MPIDI_CH3_Pkt_get_resp {
    MPIDI_CH3_Pkt_type_t type;
    MPI_Request request_handle;
    /* followings are used to decrement ack_counter at origin */
    int target_rank;
    MPI_Win source_win_handle;
    MPIDI_CH3_Pkt_flags_t flags;
} MPIDI_CH3_Pkt_get_resp_t;

538
typedef struct MPIDI_CH3_Pkt_accum {
539
    MPIDI_CH3_Pkt_type_t type;
540
    MPIDI_CH3_Pkt_flags_t flags;
541 542 543
    void *addr;
    int count;
    MPI_Datatype datatype;
544
    int dataloop_size;          /* for derived datatypes */
545
    MPI_Op op;
546 547 548 549 550 551 552
    MPI_Win target_win_handle;  /* Used in the last RMA operation in each
                                 * epoch for decrementing rma op counter in
                                 * active target rma and for unlocking window
                                 * in passive target rma. Otherwise set to NULL*/
    MPI_Win source_win_handle;  /* Used in the last RMA operation in an
                                 * epoch in the case of passive target rma
                                 * with shared locks. Otherwise set to NULL*/
553
    char data[MPIDI_RMA_IMMED_BYTES];
554 555 556
    size_t immed_len;
    int lock_type;    /* used when piggybacking LOCK message. */
    int origin_rank;  /* used when piggybacking LOCK message. */
557 558
} MPIDI_CH3_Pkt_accum_t;

Xin Zhao's avatar
Xin Zhao committed
559 560 561
typedef struct MPIDI_CH3_Pkt_get_accum {
    MPIDI_CH3_Pkt_type_t type;
    MPIDI_CH3_Pkt_flags_t flags;
562
    MPI_Request request_handle; /* For get_accumulate response */
Xin Zhao's avatar
Xin Zhao committed
563 564 565
    void *addr;
    int count;
    MPI_Datatype datatype;
566
    int dataloop_size;          /* for derived datatypes */
Xin Zhao's avatar
Xin Zhao committed
567
    MPI_Op op;
568 569 570 571 572 573 574
    MPI_Win target_win_handle;  /* Used in the last RMA operation in each
                                 * epoch for decrementing rma op counter in
                                 * active target rma and for unlocking window
                                 * in passive target rma. Otherwise set to NULL*/
    MPI_Win source_win_handle;  /* Used in the last RMA operation in an
                                 * epoch in the case of passive target rma
                                 * with shared locks. Otherwise set to NULL*/
575
    char data[MPIDI_RMA_IMMED_BYTES];
576 577 578
    size_t immed_len;
    int lock_type;     /* used when piggybacking LOCK message. */
    int origin_rank;   /* used when piggybacking LOCK message. */
Xin Zhao's avatar
Xin Zhao committed
579 580
} MPIDI_CH3_Pkt_get_accum_t;

581
typedef struct MPIDI_CH3_Pkt_get_accum_resp {
582
    MPIDI_CH3_Pkt_type_t type;
583
    MPI_Request request_handle;
584 585 586 587 588
    /* followings are used to decrement ack_counter at origin */
    int target_rank;
    MPI_Win source_win_handle;
    MPIDI_CH3_Pkt_flags_t flags;
} MPIDI_CH3_Pkt_get_accum_resp_t;
589

590
typedef struct MPIDI_CH3_Pkt_cas {
591
    MPIDI_CH3_Pkt_type_t type;
592
    MPIDI_CH3_Pkt_flags_t flags;
593
    MPI_Datatype datatype;
594
    void *addr;
595
    MPI_Request request_handle;
596 597 598 599 600
    MPI_Win source_win_handle;
    MPI_Win target_win_handle;  /* Used in the last RMA operation in each
                                 * epoch for decrementing rma op counter in
                                 * active target rma and for unlocking window
                                 * in passive target rma. Otherwise set to NULL*/
601 602
    MPIDI_CH3_CAS_Immed_u origin_data;
    MPIDI_CH3_CAS_Immed_u compare_data;
603 604
    int lock_type;     /* used when piggybacking LOCK message. */
    int origin_rank;   /* used when piggybacking LOCK message. */
605
} MPIDI_CH3_Pkt_cas_t;
606

607
typedef struct MPIDI_CH3_Pkt_cas_resp {
608 609
    MPIDI_CH3_Pkt_type_t type;
    MPI_Request request_handle;
610 611
    MPIDI_CH3_CAS_Immed_u data;
    /* followings are used to decrement ack_counter at orign */
612
    int target_rank;
613
    MPI_Win source_win_handle;
614 615
    MPIDI_CH3_Pkt_flags_t flags;
} MPIDI_CH3_Pkt_cas_resp_t;
616

617
typedef struct MPIDI_CH3_Pkt_fop {
618
    MPIDI_CH3_Pkt_type_t type;
619
    MPIDI_CH3_Pkt_flags_t flags;
620 621 622
    MPI_Datatype datatype;
    void *addr;
    MPI_Op op;
623
    MPI_Request request_handle;
624
    MPI_Win source_win_handle;
625 626 627 628
    MPI_Win target_win_handle;  /* Used in the last RMA operation in each
                                 * epoch for decrementing rma op counter in
                                 * active target rma and for unlocking window
                                 * in passive target rma. Otherwise set to NULL*/
629
    char data[MPIDI_RMA_IMMED_BYTES];
630 631 632 633
    int immed_len;
    int lock_type;     /* used when piggybacking LOCK message. */
    int origin_rank;   /* used when piggybacking LOCK message. */
} MPIDI_CH3_Pkt_fop_t;
634

635
typedef struct MPIDI_CH3_Pkt_fop_resp {
636 637
    MPIDI_CH3_Pkt_type_t type;
    MPI_Request request_handle;
638 639 640
    char data[MPIDI_RMA_IMMED_BYTES];
    int immed_len;
    /* followings are used to decrement ack_counter at orign */
641
    int target_rank;
642 643 644
    MPI_Win source_win_handle;
    MPIDI_CH3_Pkt_flags_t flags;
} MPIDI_CH3_Pkt_fop_resp_t;
645

646
typedef struct MPIDI_CH3_Pkt_lock {
647
    MPIDI_CH3_Pkt_type_t type;
648
    int lock_type;
649 650
    MPI_Win target_win_handle;
    MPI_Win source_win_handle;
651 652
    int target_rank;            /* Used in unluck/flush response to look up the
                                 * target state at the origin. */
653
    int origin_rank;
654
} MPIDI_CH3_Pkt_lock_t;
655

Xin Zhao's avatar
Xin Zhao committed
656 657
typedef struct MPIDI_CH3_Pkt_unlock {
    MPIDI_CH3_Pkt_type_t type;
658
    int lock_type;
Xin Zhao's avatar
Xin Zhao committed
659 660
    MPI_Win target_win_handle;
    MPI_Win source_win_handle;
661 662
    int target_rank;            /* Used in unluck/flush response to look up the
                                 * target state at the origin. */
663
    int origin_rank;
664
    MPIDI_CH3_Pkt_flags_t flags;
Xin Zhao's avatar
Xin Zhao committed
665 666 667 668
} MPIDI_CH3_Pkt_unlock_t;

typedef struct MPIDI_CH3_Pkt_flush {
    MPIDI_CH3_Pkt_type_t type;
669
    int lock_type;
Xin Zhao's avatar
Xin Zhao committed
670 671
    MPI_Win target_win_handle;
    MPI_Win source_win_handle;
672 673
    int target_rank;            /* Used in unluck/flush response to look up the
                                 * target state at the origin. */
674
    int origin_rank;
Xin Zhao's avatar
Xin Zhao committed
675 676
} MPIDI_CH3_Pkt_flush_t;

677
typedef struct MPIDI_CH3_Pkt_lock_granted {
678 679 680
    MPIDI_CH3_Pkt_type_t type;
    MPIDI_CH3_Pkt_flags_t flags;
    MPI_Win source_win_handle;
681 682
    int target_rank;            /* Used in flush_ack response to look up the
                                 * target state at the origin. */
683
} MPIDI_CH3_Pkt_lock_granted_t;
684

Xin Zhao's avatar
Xin Zhao committed
685 686
typedef struct MPIDI_CH3_Pkt_flush_ack {
    MPIDI_CH3_Pkt_type_t type;
687
    MPI_Win source_win_handle;
688 689 690
    int target_rank;            /* Used in flush_ack response to look up the
                                 * target state at the origin. */
    MPIDI_CH3_Pkt_flags_t flags;
Xin Zhao's avatar
Xin Zhao committed
691
} MPIDI_CH3_Pkt_flush_ack_t;
692

693 694 695 696
typedef struct MPIDI_CH3_Pkt_decr_at_counter {
    MPIDI_CH3_Pkt_type_t type;
    MPI_Win target_win_handle;
} MPIDI_CH3_Pkt_decr_at_counter_t;
697

698
typedef struct MPIDI_CH3_Pkt_close {
699 700
    MPIDI_CH3_Pkt_type_t type;
    int ack;
701
} MPIDI_CH3_Pkt_close_t;
702

703
typedef struct MPIDI_CH3_Pkt_revoke {
Wesley Bland's avatar
Wesley Bland committed
704 705
    MPIDI_CH3_Pkt_type_t type;
    MPIR_Context_id_t revoked_comm;
706
} MPIDI_CH3_Pkt_revoke_t;
Wesley Bland's avatar
Wesley Bland committed
707

708
typedef union MPIDI_CH3_Pkt {
709 710
    MPIDI_CH3_Pkt_type_t type;
    MPIDI_CH3_Pkt_eager_send_t eager_send;
711
#if defined(USE_EAGER_SHORT)
712
    MPIDI_CH3_Pkt_eagershort_send_t eagershort_send;
713
#endif                          /* defined(USE_EAGER_SHORT) */
714 715 716 717 718 719 720 721 722 723
    MPIDI_CH3_Pkt_eager_sync_send_t eager_sync_send;
    MPIDI_CH3_Pkt_eager_sync_ack_t eager_sync_ack;
    MPIDI_CH3_Pkt_eager_send_t ready_send;
    MPIDI_CH3_Pkt_rndv_req_to_send_t rndv_req_to_send;
    MPIDI_CH3_Pkt_rndv_clr_to_send_t rndv_clr_to_send;
    MPIDI_CH3_Pkt_rndv_send_t rndv_send;
    MPIDI_CH3_Pkt_cancel_send_req_t cancel_send_req;
    MPIDI_CH3_Pkt_cancel_send_resp_t cancel_send_resp;
    MPIDI_CH3_Pkt_put_t put;
    MPIDI_CH3_Pkt_get_t get;
724
    MPIDI_CH3_Pkt_get_resp_t get_resp;
725
    MPIDI_CH3_Pkt_accum_t accum;
Xin Zhao's avatar
Xin Zhao committed
726
    MPIDI_CH3_Pkt_get_accum_t get_accum;
727
    MPIDI_CH3_Pkt_lock_t lock;
728
    MPIDI_CH3_Pkt_lock_granted_t lock_granted;
729
    MPIDI_CH3_Pkt_unlock_t unlock;
730
    MPIDI_CH3_Pkt_flush_t flush;
731
    MPIDI_CH3_Pkt_flush_ack_t flush_ack;
732
    MPIDI_CH3_Pkt_decr_at_counter_t decr_at_cnt;
733
    MPIDI_CH3_Pkt_close_t close;
734 735 736 737 738
    MPIDI_CH3_Pkt_cas_t cas;
    MPIDI_CH3_Pkt_cas_resp_t cas_resp;
    MPIDI_CH3_Pkt_fop_t fop;
    MPIDI_CH3_Pkt_fop_resp_t fop_resp;
    MPIDI_CH3_Pkt_get_accum_resp_t get_accum_resp;
Wesley Bland's avatar
Wesley Bland committed
739
    MPIDI_CH3_Pkt_revoke_t revoke;
740
# if defined(MPIDI_CH3_PKT_DECL)
741
     MPIDI_CH3_PKT_DECL
742
# endif
743
} MPIDI_CH3_Pkt_t;
744 745

#if defined(MPID_USE_SEQUENCE_NUMBERS)
746
typedef struct MPIDI_CH3_Pkt_send_container {
747
    MPIDI_CH3_Pkt_send_t pkt;
748 749
    struct MPIDI_CH3_Pkt_send_container_s *next;
} MPIDI_CH3_Pkt_send_container_t;
750 751 752
#endif

#endif