refactor dping path for new swim changes

src/ssg.c

@@ -125,15 +125,13 @@ int ssg_finalize()
 
 static int ssg_get_swim_dping_target(
     void *group_data,
-    hg_addr_t *target_addr,
-    swim_member_state_t *target_swim_ms);
+    swim_dping_target_info_t *target_info);
 static void ssg_gen_rand_member_list(
     ssg_group_t *g);
 
 static int ssg_get_swim_dping_target(
     void *group_data,
-    hg_addr_t *target_addr,
-    swim_member_state_t *target_swim_ms)
+    swim_dping_target_info_t *target_info)
 {
     ssg_group_t *g = (ssg_group_t *)group_data;
     ssg_member_state_t *target_ms;
@@ -147,10 +145,9 @@ static int ssg_get_swim_dping_target(
     /* pull random member off head of list and return addr */
     target_ms = g->member_list;
     LL_DELETE(g->member_list, target_ms);
-    *target_addr = target_ms->addr;
-    *target_swim_ms = target_ms->swim_state;
-
-    printf("%lu: pinging %lu\n", g->self_id, target_ms->id);
+    target_info->id = (swim_member_id_t)target_ms->id;
+    target_info->addr = target_ms->addr;
+    target_info->swim_state = target_ms->swim_state;
 
     return 0;
 }
@@ -236,7 +233,8 @@ ssg_group_id_t ssg_group_create(
     swim_group_mgmt_callbacks_t swim_callbacks = {
         .get_dping_target = &ssg_get_swim_dping_target,
     };
-    g->swim_ctx = swim_init(ssg_inst->mid, g, swim_callbacks, 1);
+    g->swim_ctx = swim_init(ssg_inst->mid, g, (swim_member_id_t)g->self_id,
+        swim_callbacks, 1);
     if (g->swim_ctx == NULL) goto fini;
 
     /* everything successful -- set the output group identifier, which is just

src/swim-fd/swim-fd-internal.h

@@ -23,12 +23,31 @@ extern "C" {
 #define SWIM_MAX_PIGGYBACK_ENTRIES      8
 #define SWIM_MAX_PIGGYBACK_TX_COUNT     50
 
+/* debug printing macro for SSG */
+#ifdef DEBUG
+#define SWIM_DEBUG(__swim_ctx, __fmt, ...) do { \
+    double __now = ABT_get_wtime(); \
+    fprintf(stdout, "%.6lf <%020"PRIu64">: SWIM: " __fmt, __now, \
+        __swim_ctx->self_id, ## __VA_ARGS__); \
+    fflush(stdout); \
+} while(0)
+#else
+#define SWIM_DEBUG(__swim_ctx, __fmt, ...) do { \
+} while(0)
+#endif
+
 /* internal swim context implementation */
 struct swim_context
 {
     margo_instance_id mid;
     /* void pointer to user group data */
     void *group_data;
+    /* XXX other state */
+    swim_member_id_t self_id;
+    swim_member_inc_nr_t self_inc_nr;
+    swim_dping_target_info_t dping_target_info;
+    int dping_target_acked;
+    double dping_timeout;
     /* XXX group mgmt callbacks */
     swim_group_mgmt_callbacks_t swim_callbacks;
     /* argobots pool for launching SWIM threads */
@@ -41,10 +60,6 @@ struct swim_context
     int prot_subgroup_sz;
     /* SWIM internal state */
     int shutdown_flag;
-    hg_addr_t dping_target_addr;
-    swim_member_state_t dping_target_state;
-    int dping_target_acked;
-    double dping_timeout;
     hg_addr_t iping_subgroup_addrs[SWIM_MAX_SUBGROUP_SIZE];
 #if 0
     /* current membership state */
@@ -62,12 +77,14 @@ struct swim_member_update
     swim_member_status_t status;
     swim_member_inc_nr_t inc_nr;
 };
+#endif
 
 /* SWIM ping function prototypes */
 void swim_register_ping_rpcs(
-    ssg_group_t * g);
+    swim_context_t * swim_ctx);
 void swim_dping_send_ult(
     void * t_arg);
+#if 0
 void swim_iping_send_ult(
     void * t_arg);
 

src/swim-fd/swim-fd.c

@@ -3,7 +3,6 @@
  *
  * See COPYRIGHT in top-level directory.
  */
-
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
@@ -57,6 +56,7 @@ static void swim_add_recent_member_update(
 swim_context_t * swim_init(
     margo_instance_id mid,
     void * group_data,
+    swim_member_id_t self_id,
     swim_group_mgmt_callbacks_t swim_callbacks,
     int active)
 {
@@ -69,11 +69,12 @@ swim_context_t * swim_init(
     memset(swim_ctx, 0, sizeof(*swim_ctx));
     swim_ctx->mid = mid;
     swim_ctx->group_data = group_data;
+    swim_ctx->self_id = self_id;
+    swim_ctx->self_inc_nr = 0;
     swim_ctx->swim_callbacks = swim_callbacks;
 
     /* initialize SWIM context */
     margo_get_handler_pool(swim_ctx->mid, &swim_ctx->swim_pool);
-    swim_ctx->dping_target_addr = HG_ADDR_NULL;
     for(i = 0; i < SWIM_MAX_SUBGROUP_SIZE; i++)
         swim_ctx->iping_subgroup_addrs[i] = HG_ADDR_NULL;
 
@@ -82,10 +83,7 @@ swim_context_t * swim_init(
     swim_ctx->prot_susp_timeout = SWIM_DEF_SUSPECT_TIMEOUT;
     swim_ctx->prot_subgroup_sz = SWIM_DEF_SUBGROUP_SIZE;
 
-    /* XXX */
-#if 0
-    swim_register_ping_rpcs(g);
-#endif
+    swim_register_ping_rpcs(swim_ctx);
 
     if(active)
     {
@@ -110,11 +108,11 @@ static void swim_prot_ult(
 
     assert(swim_ctx != NULL);
 
-#if 0
-    SSG_DEBUG(g, "SWIM: protocol start (period_len=%.4f, susp_timeout=%d, subgroup_size=%d)\n",
+    SWIM_DEBUG(swim_ctx,
+        "protocol start (period_len=%.4f, susp_timeout=%d, subgroup_size=%d)\n",
         swim_ctx->prot_period_len, swim_ctx->prot_susp_timeout,
         swim_ctx->prot_subgroup_sz);
-#endif
+
     while(!(swim_ctx->shutdown_flag))
     {
         /* spawn a ULT to run this tick */
@@ -128,9 +126,9 @@ static void swim_prot_ult(
         /* sleep for a protocol period length */
         margo_thread_sleep(swim_ctx->mid, swim_ctx->prot_period_len);
     }
-#if 0
-    SSG_DEBUG(g, "SWIM: protocol shutdown\n");
-#endif
+
+    SWIM_DEBUG(swim_ctx, "protocol shutdown\n");
+
     return;
 }
 
@@ -160,23 +158,21 @@ static void swim_tick_ult(
 #endif
 
     /* pick a random member from view and ping */
-    ret = swim_ctx->swim_callbacks.get_dping_target(swim_ctx->group_data,
-        &swim_ctx->dping_target_addr, &swim_ctx->dping_target_state);
+    ret = swim_ctx->swim_callbacks.get_dping_target(
+            swim_ctx->group_data,
+            &swim_ctx->dping_target_info);
     if(ret != 0)
     {
         /* no available members, back out */
-#if 0
-        SSG_DEBUG(g, "SWIM: no group members available to dping\n");
-#endif
+        SWIM_DEBUG(swim_ctx, "no group members available to dping\n");
         return;
     }
 
     /* TODO: calculate estimated RTT using sliding window of past RTTs */
     swim_ctx->dping_timeout = 250.0;
 
-#if 0
     /* kick off dping request ULT */
-    swim_ctx->ping_target_acked = 0;
+    swim_ctx->dping_target_acked = 0;
     ret = ABT_thread_create(swim_ctx->swim_pool, swim_dping_send_ult, swim_ctx,
         ABT_THREAD_ATTR_NULL, NULL);
     if(ret != ABT_SUCCESS)
@@ -186,8 +182,9 @@ static void swim_tick_ult(
     }
 
     /* sleep for an RTT and wait for an ack for this dping req */
-    margo_thread_sleep(ssg_inst->mid, swim_ctx->dping_timeout);
+    margo_thread_sleep(swim_ctx->mid, swim_ctx->dping_timeout);
 
+#if 0
     /* if we don't hear back from the target after an RTT, kick off
      * a set of indirect pings to a subgroup of group members
      */

src/swim-fd/swim-fd.h

@@ -17,6 +17,7 @@ extern "C" {
 typedef struct swim_context swim_context_t;
 
 /* swim member specific types */
+typedef uint64_t swim_member_id_t;
 typedef uint32_t swim_member_inc_nr_t;
 typedef enum swim_member_status
 {
@@ -31,6 +32,13 @@ typedef struct swim_member_state
     swim_member_status_t status;
 } swim_member_state_t;
 
+typedef struct swim_dping_target_info
+{
+    swim_member_id_t id;
+    hg_addr_t addr;
+    swim_member_state_t swim_state;
+} swim_dping_target_info_t;
+
 #define SWIM_MEMBER_STATE_INIT(__ms) do { \
     __ms.inc_nr = 0; \
     __ms.status = SWIM_MEMBER_ALIVE; \
@@ -41,8 +49,7 @@ typedef struct swim_group_mgmt_callbacks
 {
     int (*get_dping_target)(
             void *group_data,
-            hg_addr_t *target_addr,
-            swim_member_state_t *target_ms
+            swim_dping_target_info_t *target_info
             );
     /* get_rand_iping_subgroup */
 } swim_group_mgmt_callbacks_t;
@@ -51,6 +58,7 @@ typedef struct swim_group_mgmt_callbacks
 swim_context_t * swim_init(
     margo_instance_id mid,
     void * group_data,
+    swim_member_id_t self_id,
     swim_group_mgmt_callbacks_t swim_callbacks,
     int active);