portals4: add macro for safe PtlMEAppend

Signed-off-by: Antonio J. Pena <apenya@mcs.anl.gov>

src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_impl.h

@@ -208,6 +208,26 @@ int MPID_nem_ptl_lmt_handle_cookie(MPIDI_VC_t *vc, MPID_Request *req, MPID_IOV s
 int MPID_nem_ptl_lmt_done_send(MPIDI_VC_t *vc, MPID_Request *req);
 int MPID_nem_ptl_lmt_done_recv(MPIDI_VC_t *vc, MPID_Request *req);
 
+/* a safe PtlMEAppend for when there is no space available */
+static inline int MPID_nem_ptl_me_append(ptl_handle_ni_t  ni_handle,
+                                         ptl_pt_index_t   pt_index,
+                                         const ptl_me_t  *me,
+                                         ptl_list_t       ptl_list,
+                                         void            *user_ptr,
+                                         ptl_handle_me_t *me_handle)
+{
+    int ret;
+
+    while (1) {
+        ret = PtlMEAppend(ni_handle, pt_index, me, ptl_list, user_ptr, me_handle);
+        if (ret != PTL_NO_SPACE)
+            break;
+        MPID_nem_ptl_poll(1);
+    }
+
+    return ret;
+}
+
 /* packet handlers */
 
 int MPID_nem_ptl_pkt_cancel_send_req_handler(MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,

src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_nm.c

@@ -63,8 +63,8 @@ int MPID_nem_ptl_nm_init(void)
         mes[i].match_bits = CTL_TAG;
         mes[i].ignore_bits = 0;
 
-        ret = PtlMEAppend(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_control_pt, &mes[i],
-                          PTL_PRIORITY_LIST, (void *)(uint64_t)i, &me_handles[i]);
+        ret = MPID_nem_ptl_me_append(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_control_pt, &mes[i],
+                                     PTL_PRIORITY_LIST, (void *)(uint64_t)i, &me_handles[i]);
         MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmeappend", "**ptlmeappend %s",
                              MPID_nem_ptl_strerror(ret));
     }
@@ -134,15 +134,8 @@ static inline int meappend_large(ptl_process_t id, MPID_Request *req, ptl_match_
 
         ++REQ_PTL(req)->num_gets;
 
-        /* if there is no space to append the entry, process outstanding events and try again */
-        while (1) {
-            ret = PtlMEAppend(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_control_pt, &me, PTL_PRIORITY_LIST, req,
-                              &foo_me_handle);
-            if (ret != PTL_NO_SPACE)
-                break;
-            MPID_nem_ptl_poll(1);
-        }
-
+        ret = MPID_nem_ptl_me_append(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_control_pt, &me, PTL_PRIORITY_LIST, req,
+                                     &foo_me_handle);
         MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmeappend", "**ptlmeappend %s",
                              MPID_nem_ptl_strerror(ret));
         MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, "PtlMEAppend(req=%p tag=%#lx)", req, tag));
@@ -450,14 +443,8 @@ int MPID_nem_ptl_nm_ctl_event_handler(const ptl_event_t *e)
             }
 
             /* Repost the recv buffer */
-            /* if there is no space to append the entry, process outstanding events and try again */
-            while (1) {
-                ret = PtlMEAppend(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_control_pt, &mes[buf_idx],
-                                  PTL_PRIORITY_LIST, e->user_ptr /* buf_idx */, &me_handles[buf_idx]);
-                if (ret != PTL_NO_SPACE)
-                    break;
-                MPID_nem_ptl_poll(1);
-            }
+            ret = MPID_nem_ptl_me_append(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_control_pt, &mes[buf_idx],
+                                         PTL_PRIORITY_LIST, e->user_ptr /* buf_idx */, &me_handles[buf_idx]);
             MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmeappend",
                                  "**ptlmeappend %s", MPID_nem_ptl_strerror(ret));
         }

src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_poll.c

@@ -106,13 +106,8 @@ static int append_overflow(int i)
     me.min_free = PTL_MAX_EAGER;
     
     /* if there is no space to append the entry, process outstanding events and try again */
-    while (1) {
-        ret = PtlMEAppend(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_pt, &me, PTL_OVERFLOW_LIST, (void *)(size_t)i,
-                          &overflow_me_handle[i]);
-        if (ret != PTL_NO_SPACE)
-            break;
-        MPID_nem_ptl_poll(1);
-    }
+    ret = MPID_nem_ptl_me_append(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_pt, &me, PTL_OVERFLOW_LIST, (void *)(size_t)i,
+                                 &overflow_me_handle[i]);
     MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmeappend", "**ptlmeappend %s", MPID_nem_ptl_strerror(ret));
 
  fn_exit:

src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_recv.c

@@ -543,13 +543,7 @@ int MPID_nem_ptl_recv_posted(MPIDI_VC_t *vc, MPID_Request *rreq)
         
     }
 
-    /* if there is no space to append the entry, process outstanding events and try again */
-    while (1) {
-        ret = PtlMEAppend(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_pt, &me, PTL_PRIORITY_LIST, rreq, &REQ_PTL(rreq)->put_me);
-        if (ret != PTL_NO_SPACE)
-            break;
-        MPID_nem_ptl_poll(1);
-    }
+    ret = MPID_nem_ptl_me_append(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_pt, &me, PTL_PRIORITY_LIST, rreq, &REQ_PTL(rreq)->put_me);
     MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmeappend", "**ptlmeappend %s", MPID_nem_ptl_strerror(ret));
     DBG_MSG_MEAPPEND("REG", vc ? vc->pg_rank : MPI_ANY_SOURCE, me, rreq);
     MPIU_DBG_MSG_P(CH3_CHANNEL, VERBOSE, "    buf=%p", me.start);

src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_send.c

@@ -41,13 +41,8 @@ static void big_meappend(void *buf, ptl_size_t left_to_send, MPIDI_VC_t *vc, ptl
         else
             me.length = left_to_send;
 
-        /* if there is no space to append the entry, process outstanding events and try again */
-        while (1) {
-            ret = PtlMEAppend(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_get_pt, &me, PTL_PRIORITY_LIST, sreq, &REQ_PTL(sreq)->get_me_p[i]);
-            if (ret != PTL_NO_SPACE)
-                break;
-            MPID_nem_ptl_poll(1);
-        }
+        ret = MPID_nem_ptl_me_append(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_get_pt, &me, PTL_PRIORITY_LIST, sreq,
+                                     &REQ_PTL(sreq)->get_me_p[i]);
         DBG_MSG_MEAPPEND("CTL", vc->pg_rank, me, sreq);
         MPIU_Assert(ret == 0);
 
@@ -387,14 +382,8 @@ static int send_msg(ptl_hdr_data_t ssend_flag, struct MPIDI_VC *vc, const void *
                 MPIU_CHKPMEM_MALLOC(REQ_PTL(sreq)->get_me_p, ptl_handle_me_t *, sizeof(ptl_handle_me_t), mpi_errno, "get_me_p");
 
                 REQ_PTL(sreq)->num_gets = 1;
-                /* if there is no space to append the entry, process outstanding events and try again */
-                while (1) {
-                    ret = PtlMEAppend(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_get_pt, &me, PTL_PRIORITY_LIST, sreq,
-                                      &REQ_PTL(sreq)->get_me_p[0]);
-                    if (ret != PTL_NO_SPACE)
-                        break;
-                    MPID_nem_ptl_poll(1);
-                }
+                ret = MPID_nem_ptl_me_append(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_get_pt, &me, PTL_PRIORITY_LIST, sreq,
+                                             &REQ_PTL(sreq)->get_me_p[0]);
                 MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmeappend", "**ptlmeappend %s", MPID_nem_ptl_strerror(ret));
                 DBG_MSG_MEAPPEND("CTL", vc->pg_rank, me, sreq);