portals4: tuning

Changes the value of various static limits in the Portals4 netmod, based
on experimentation results and suggestions from collaborators.

1. Bump most ni_limits from 32K to 64K. These limits relate closely to
   queue depth. We can reasonably expect to support a queue depth
   of 64K.

2. Limit issued origin events to 500. This translates to sending ~250
   operations to Portals at a time, which over IB is roughly the
   saturation point. TODO: turn this into a CVAR.

3. Limit per target issued operations to 50. This will give the target a
   better chance to process events without being overwhelmed by a single
   process. TODO: turn this into a CVAR, also.

4. Allocate more buffer space for incoming control messages. Observed
   results, especially with larger messages, showed that more buffer space
   cuts down on flow-control events.
Signed-off-by: Antonio J. Pena <apenya@mcs.anl.gov>

src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_init.c

@@ -12,10 +12,13 @@
 #error Checkpointing not implemented
 #endif
 
-#define UNEXPECTED_HDR_COUNT 32768
-#define EVENT_COUNT          32768
-#define LIST_SIZE            32768
-#define ENTRY_COUNT          32768
+#define UNEXPECTED_HDR_COUNT (1024*64)
+#define EVENT_COUNT          (1024*64)
+#define LIST_SIZE            (1024*64)
+#define MAX_ENTRIES          (1024*64)
+#define ENTRY_COUNT          (1024*64)
+/* FIXME: turn ORIGIN_EVENTS into a CVAR */
+#define ORIGIN_EVENTS        (500)
 #define NID_KEY  "NID"
 #define PID_KEY  "PID"
 #define PTI_KEY  "PTI"
@@ -245,7 +248,7 @@ static int ptl_init(MPIDI_PG_t *pg_p, int pg_rank, char **bc_val_p, int *val_max
     MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmdbind", "**ptlmdbind %s", MPID_nem_ptl_strerror(ret));
 
     /* currently, rportlas only works with a single NI and EQ */
-    ret = MPID_nem_ptl_rptl_init(MPIDI_Process.my_pg->size, EVENT_COUNT, get_target_info);
+    ret = MPID_nem_ptl_rptl_init(MPIDI_Process.my_pg->size, ORIGIN_EVENTS, get_target_info);
     MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlniinit", "**ptlniinit %s", MPID_nem_ptl_strerror(ret));
 
     /* allow rportal to manage the primary portal and retransmit if needed */

src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_nm.c

@@ -8,8 +8,8 @@
 #include <mpl_utlist.h>
 #include "rptl.h"
 
-#define NUM_RECV_BUFS 8
-#define BUFSIZE (1024*1024)
+#define NUM_RECV_BUFS 2
+#define BUFSIZE (1024*1024*50)
 #define CTL_TAG 0
 #define GET_TAG 1
 #define PAYLOAD_SIZE  (PTL_MAX_EAGER - sizeof(MPIDI_CH3_Pkt_t))

src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_poll.c

@@ -103,7 +103,7 @@ static int append_overflow(int i)
     me.match_id = id_any;
     me.match_bits = 0;
     me.ignore_bits = ~((ptl_match_bits_t)0);
-    me.min_free = PTL_MAX_EAGER;
+    me.min_free = PTL_LARGE_THRESHOLD;
     
     /* if there is no space to append the entry, process outstanding events and try again */
     ret = PtlMEAppend(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_pt, &me, PTL_OVERFLOW_LIST, (void *)(size_t)i,

src/mpid/ch3/channels/nemesis/netmod/portals4/rptl.c

@@ -7,6 +7,10 @@
 #include "ptl_impl.h"
 #include "rptl.h"
 
+/* FIXME: turn this into a CVAR, or fraction of the event limit from
+   rptl_init */
+#define PER_TARGET_THRESHOLD 50
+
 /*
  * Prereqs:
  *
@@ -93,6 +97,7 @@ static int find_target(ptl_process_t id, struct rptl_target **target)
         t->op_pool = NULL;
         t->data_op_list = NULL;
         t->control_op_list = NULL;
+        t->issued_data_ops = 0;
     }
 
     *target = t;
@@ -220,7 +225,7 @@ static int poke_progress(void)
             /* we should not get any NACKs on the control portal */
             assert(op->state != RPTL_OP_STATE_NACKED);
 
-            if (rptl_info.origin_events_left < 2) {
+            if (rptl_info.origin_events_left < 2 || target->issued_data_ops > PER_TARGET_THRESHOLD) {
                 /* too few origin events left.  we can't issue this op
                  * or any following op to this target in order to
                  * maintain ordering */
@@ -228,6 +233,7 @@ static int poke_progress(void)
             }
 
             rptl_info.origin_events_left -= 2;
+            target->issued_data_ops++;
 
             /* force request for an ACK even if the user didn't ask
              * for it.  replace the user pointer with the OP id. */
@@ -255,7 +261,7 @@ static int poke_progress(void)
                 if (op->state == RPTL_OP_STATE_NACKED)
                     break;
 
-                if (rptl_info.origin_events_left < 2) {
+                if (rptl_info.origin_events_left < 2 || target->issued_data_ops > PER_TARGET_THRESHOLD) {
                     /* too few origin events left.  we can't issue
                      * this op or any following op to this target in
                      * order to maintain ordering */
@@ -263,6 +269,7 @@ static int poke_progress(void)
                 }
 
                 rptl_info.origin_events_left -= 2;
+                target->issued_data_ops++;
 
                 /* force request for an ACK even if the user didn't
                  * ask for it.  replace the user pointer with the OP
@@ -283,7 +290,7 @@ static int poke_progress(void)
                 if (op->state == RPTL_OP_STATE_NACKED)
                     break;
 
-                if (rptl_info.origin_events_left < 1) {
+                if (rptl_info.origin_events_left < 1 || target->issued_data_ops > PER_TARGET_THRESHOLD) {
                     /* too few origin events left.  we can't issue
                      * this op or any following op to this target in
                      * order to maintain ordering */
@@ -291,6 +298,7 @@ static int poke_progress(void)
                 }
 
                 rptl_info.origin_events_left--;
+                target->issued_data_ops++;
 
                 ret = PtlGet(op->u.get.md_handle, op->u.get.local_offset, op->u.get.length,
                              op->u.get.target_id, op->u.get.pt_index, op->u.get.match_bits,
@@ -539,6 +547,7 @@ static int get_event_info(ptl_event_t * event, struct rptl **ret_rptl, struct rp
         op = (struct rptl_op *) event->user_ptr;
 
         rptl_info.origin_events_left++;
+        op->target->issued_data_ops--;
 
         /* see if there are any pending ops to be issued */
         ret = poke_progress();

src/mpid/ch3/channels/nemesis/netmod/portals4/rptl.h

@@ -158,6 +158,8 @@ struct rptl_target {
 
     struct rptl_target *next;
     struct rptl_target *prev;
+
+    int issued_data_ops;
 };
 
 struct rptl_info {