Redefined RMA extended packet header in CH3 layer.

This patch redefined RMA extended header in CH3 layer based on commit
25e40e43. The extended header helps RMA issue RMA-specific header. Each
OP may define multiple attributes dynamically, or just let it empty.
Here is a summary of the detailed implementation.

(1) We define a packet type for every kind of extended header, and for
every kind of OP. For now, we have defined stream_{acc|get_acc},
derived_{put|get|acc|get_acc} and stream_derived_{acc|get_acc}.

(2) Extended header may contain fixed-attributes, or variable-length
parts (i.e., dataloop). We define all fixed-attributes in packet
structure, and followed by variable-length parts.
For example:
-------------------------------------------------------------------
| fixed attributes... | variable-len part 1 | variable-len part 2 |
-------------------------------------------------------------------

(3) Origin process simply allocates a contig buffer to fill both fixed part
and variable-len parts, and transfer it to netmod through req; target
process can specify separate buffers to receive the variable-len parts
from netmod in order to avoid extra copy (i.e., dataloop).

(4) Each OP has different initialization and packet-handler on origin
and target side respectively. For now ACC and GET_ACC share generic
routine since all of their attributes are the same.
Signed-off-by: Xin Zhao <xinzhao3@illinois.edu>
Signed-off-by: Pavan Balaji <balaji@anl.gov>

src/mpid/ch3/channels/nemesis/include/mpid_nem_inline.h

@@ -353,6 +353,9 @@ MPID_nem_mpich_sendv_header (MPID_IOV **iov, int *n_iov,
     buf_offset += sizeof(MPIDI_CH3_Pkt_t);
 
     if (ext_hdr_sz > 0) {
+        /* ensure extended header fits in this cell. */
+        MPIU_Assert(MPID_NEM_MPICH_DATA_LEN - buf_offset >= ext_hdr_sz);
+
         /* when extended packet header exists, copy it */
         MPIU_Memcpy((void *)((char *)(el->pkt.mpich.p.payload) + buf_offset), ext_hdr_ptr, ext_hdr_sz);
         buf_offset += ext_hdr_sz;

src/mpid/ch3/include/mpid_rma_oplist.h

@@ -66,7 +66,6 @@ static inline MPIDI_RMA_Op_t *MPIDI_CH3I_Win_op_alloc(MPID_Win * win_ptr)
         MPL_LL_DELETE(win_ptr->op_pool_head, win_ptr->op_pool_tail, e);
     }
 
-    e->dataloop = NULL;
     e->single_req = NULL;
     e->multi_reqs = NULL;
     e->reqs_size = 0;
@@ -90,11 +89,6 @@ static inline int MPIDI_CH3I_Win_op_free(MPID_Win * win_ptr, MPIDI_RMA_Op_t * e)
 {
     int mpi_errno = MPI_SUCCESS;
 
-    /* Check if we allocated a dataloop for this op (see send/recv_rma_msg) */
-    if (e->dataloop != NULL) {
-        MPIU_Free(e->dataloop);
-    }
-
     /* We enqueue elements to the right pool, so when they get freed
      * at window free time, they won't conflict with the global pool
      * or other windows */

src/mpid/ch3/include/mpid_rma_types.h

@@ -23,23 +23,6 @@ enum MPIDI_RMA_Datatype {
  * a Request.
  */
 
-/* to send derived datatype across in RMA ops */
-typedef struct MPIDI_RMA_dtype_info {   /* for derived datatypes */
-    int is_contig;
-    MPI_Aint max_contig_blocks;
-    MPI_Aint size;
-    MPI_Aint extent;
-    MPI_Aint dataloop_size;     /* not needed because this info is sent in
-                                 * packet header. remove it after lock/unlock
-                                 * is implemented in the device */
-    void *dataloop;             /* pointer needed to update pointers
-                                 * within dataloop on remote side */
-    int dataloop_depth;
-    int basic_type;
-    MPI_Aint ub, lb, true_ub, true_lb;
-    int has_sticky_ub, has_sticky_lb;
-} MPIDI_RMA_dtype_info;
-
 typedef enum MPIDI_RMA_Pool_type {
     MPIDI_RMA_POOL_WIN = 6,
     MPIDI_RMA_POOL_GLOBAL = 7
@@ -66,9 +49,6 @@ typedef struct MPIDI_RMA_Op {
                                  * when reqs_size == 1, single_req is used;
                                  * when reqs_size > 1, multi_reqs is used. */
 
-    MPIDI_RMA_dtype_info dtype_info;
-    void *dataloop;
-
     int target_rank;
 
     MPIDI_CH3_Pkt_t pkt;

src/mpid/ch3/include/mpidpkt.h

@@ -863,13 +863,60 @@ typedef union MPIDI_CH3_Pkt {
 #endif
 } MPIDI_CH3_Pkt_t;
 
-typedef struct MPIDI_CH3_Ext_pkt_accum {
+/* Extended header packet types */
+
+/* to send derived datatype across in RMA ops */
+typedef struct MPIDI_RMA_dtype_info {   /* for derived datatypes */
+    int is_contig;
+    MPI_Aint max_contig_blocks;
+    MPI_Aint size;
+    MPI_Aint extent;
+    MPI_Aint dataloop_size;     /* not needed because this info is sent in
+                                 * packet header. remove it after lock/unlock
+                                 * is implemented in the device */
+    void *dataloop;             /* pointer needed to update pointers
+                                 * within dataloop on remote side */
+    int dataloop_depth;
+    int basic_type;
+    MPI_Aint ub, lb, true_ub, true_lb;
+    int has_sticky_ub, has_sticky_lb;
+} MPIDI_RMA_dtype_info;
+
+typedef struct MPIDI_CH3_Ext_pkt_stream {
     MPI_Aint stream_offset;
-} MPIDI_CH3_Ext_pkt_accum_t;
-
-typedef struct MPIDI_CH3_Ext_pkt_get_accum {
+} MPIDI_CH3_Ext_pkt_stream_t;
+
+typedef struct MPIDI_CH3_Ext_pkt_derived {
+    MPIDI_RMA_dtype_info dtype_info;
+    /* Follow with variable-length dataloop.
+     * On origin we allocate a large buffer including
+     * this header and the dataloop; on target we use
+     * separate buffer to receive dataloop in order
+     * to avoid extra copy.*/
+} MPIDI_CH3_Ext_pkt_derived_t;
+
+typedef struct MPIDI_CH3_Ext_pkt_stream_derived {
     MPI_Aint stream_offset;
-} MPIDI_CH3_Ext_pkt_get_accum_t;
+    MPIDI_RMA_dtype_info dtype_info;
+    /* follow with variable-length dataloop. */
+} MPIDI_CH3_Ext_pkt_stream_derived_t;
+
+/* Note that since ACC and GET_ACC contain the same extended attributes,
+ * we use generic routines for them in some places (see below).
+ * If we add OP-specific attribute in future, we should handle them separately.
+ *  1. origin issuing function
+ *  2. target packet handler
+ *  3. target data receive complete handler. */
+typedef MPIDI_CH3_Ext_pkt_stream_t MPIDI_CH3_Ext_pkt_accum_stream_t;
+typedef MPIDI_CH3_Ext_pkt_derived_t MPIDI_CH3_Ext_pkt_accum_derived_t;
+typedef MPIDI_CH3_Ext_pkt_stream_derived_t MPIDI_CH3_Ext_pkt_accum_stream_derived_t;
+
+typedef MPIDI_CH3_Ext_pkt_stream_t MPIDI_CH3_Ext_pkt_get_accum_stream_t;
+typedef MPIDI_CH3_Ext_pkt_derived_t MPIDI_CH3_Ext_pkt_get_accum_derived_t;
+typedef MPIDI_CH3_Ext_pkt_stream_derived_t MPIDI_CH3_Ext_pkt_get_accum_stream_derived_t;
+
+typedef MPIDI_CH3_Ext_pkt_derived_t MPIDI_CH3_Ext_pkt_put_derived_t;
+typedef MPIDI_CH3_Ext_pkt_derived_t MPIDI_CH3_Ext_pkt_get_derived_t;
 
 #if defined(MPID_USE_SEQUENCE_NUMBERS)
 typedef struct MPIDI_CH3_Pkt_send_container {

src/mpid/ch3/include/mpidpre.h

@@ -424,8 +424,7 @@ typedef struct MPIDI_Request {
     MPI_Op op;
     /* For accumulate, since data is first read into a tmp_buf */
     void *real_user_buf;
-    /* For derived datatypes at target */
-    struct MPIDI_RMA_dtype_info *dtype_info;
+    /* For derived datatypes at target. */
     void *dataloop;
     /* req. handle needed to implement derived datatype gets.
      * It also used for remembering user request of request-based RMA operations. */
@@ -436,7 +435,9 @@ typedef struct MPIDI_Request {
     struct MPIDI_RMA_Target_lock_entry *target_lock_queue_entry;
     MPI_Request resp_request_handle; /* Handle for get_accumulate response */
 
-    void *ext_hdr_ptr; /* pointer to extended packet header */
+    void *ext_hdr_ptr; /* Pointer to extended packet header.
+                        * It is allocated in RMA issuing/pkt_handler functions,
+                        * and freed when release request. */
     MPIDI_msg_sz_t ext_hdr_sz;
 
     MPIDI_REQUEST_SEQNUM

src/mpid/ch3/include/mpidrma.h

@@ -399,13 +399,15 @@ static inline int enqueue_lock_origin(MPID_Win * win_ptr, MPIDI_VC_t * vc,
             MPIU_Assert(pkt->type == MPIDI_CH3_PKT_ACCUMULATE ||
                         pkt->type == MPIDI_CH3_PKT_GET_ACCUM);
 
+            /* Only basic datatype may contain piggyback lock.
+             * Thus we do not check extended header type for derived case.*/
             if (pkt->type == MPIDI_CH3_PKT_ACCUMULATE) {
-                recv_data_sz += sizeof(MPIDI_CH3_Ext_pkt_accum_t);
-                buf_size += sizeof(MPIDI_CH3_Ext_pkt_accum_t);
+                recv_data_sz += sizeof(MPIDI_CH3_Ext_pkt_accum_stream_t);
+                buf_size += sizeof(MPIDI_CH3_Ext_pkt_accum_stream_t);
             }
             else {
-                recv_data_sz += sizeof(MPIDI_CH3_Ext_pkt_get_accum_t);
-                buf_size += sizeof(MPIDI_CH3_Ext_pkt_get_accum_t);
+                recv_data_sz += sizeof(MPIDI_CH3_Ext_pkt_get_accum_stream_t);
+                buf_size += sizeof(MPIDI_CH3_Ext_pkt_get_accum_stream_t);
             }
         }
 
@@ -1157,4 +1159,28 @@ static inline int poke_progress_engine(void)
     goto fn_exit;
 }
 
+static inline void MPIDI_CH3_ExtPkt_Accum_get_stream(MPIDI_CH3_Pkt_flags_t flags,
+                                                     int is_derived_dt, void *ext_hdr_ptr,
+                                                     MPI_Aint * stream_offset)
+{
+    if ((flags & MPIDI_CH3_PKT_FLAG_RMA_STREAM) && is_derived_dt) {
+        MPIU_Assert(ext_hdr_ptr != NULL);
+        (*stream_offset) =
+            ((MPIDI_CH3_Ext_pkt_accum_stream_derived_t *) ext_hdr_ptr)->stream_offset;
+    }
+    else if (flags & MPIDI_CH3_PKT_FLAG_RMA_STREAM) {
+        MPIU_Assert(ext_hdr_ptr != NULL);
+        (*stream_offset) = ((MPIDI_CH3_Ext_pkt_accum_stream_t *) ext_hdr_ptr)->stream_offset;
+    }
+}
+
+static inline void MPIDI_CH3_ExtPkt_Gaccum_get_stream(MPIDI_CH3_Pkt_flags_t flags,
+                                                      int is_derived_dt, void *ext_hdr_ptr,
+                                                      MPI_Aint * stream_offset)
+{
+    /* We do not check packet match here, because error must have already been
+     * reported at header init time (on origin) and at packet receive time (on target).  */
+    return MPIDI_CH3_ExtPkt_Accum_get_stream(flags, is_derived_dt, ext_hdr_ptr, stream_offset);
+}
+
 #endif /* MPID_RMA_H_INCLUDED */

src/mpid/ch3/src/ch3u_request.c

@@ -84,7 +84,6 @@ MPID_Request * MPID_Request_create(void)
 	req->dev.target_win_handle = MPI_WIN_NULL;
 	req->dev.source_win_handle = MPI_WIN_NULL;
         req->dev.target_lock_queue_entry = NULL;
-	req->dev.dtype_info	   = NULL;
 	req->dev.dataloop	   = NULL;
 	req->dev.iov_offset        = 0;
         req->dev.flags             = MPIDI_CH3_PKT_FLAG_NONE;