more swim updates

src/ssg.c

@@ -125,13 +125,30 @@ int ssg_finalize()
 
 static int ssg_get_swim_dping_target(
     void *group_data,
-    swim_dping_target_info_t *target_info);
+    swim_member_id_t *target_id,
+    swim_member_inc_nr_t *target_inc_nr,
+    hg_addr_t *target_addr);
+static int ssg_get_swim_iping_targets(
+    void *group_data,
+    swim_member_id_t *target_ids,
+    hg_addr_t *target_addrs);
+static void ssg_get_swim_member_addr(
+    void *group_data,
+    swim_member_id_t id,
+    hg_addr_t *target_addr);
+static void ssg_get_swim_member_state(
+    void *group_data,
+    swim_member_id_t id,
+    swim_member_state_t *state);
+
 static void ssg_gen_rand_member_list(
     ssg_group_t *g);
 
 static int ssg_get_swim_dping_target(
     void *group_data,
-    swim_dping_target_info_t *target_info)
+    swim_member_id_t *target_id,
+    swim_member_inc_nr_t *target_inc_nr,
+    hg_addr_t *target_addr)
 {
     ssg_group_t *g = (ssg_group_t *)group_data;
     ssg_member_state_t *target_ms;
@@ -145,9 +162,21 @@ static int ssg_get_swim_dping_target(
     /* pull random member off head of list and return addr */
     target_ms = g->member_list;
     LL_DELETE(g->member_list, target_ms);
-    target_info->id = (swim_member_id_t)target_ms->id;
-    target_info->addr = target_ms->addr;
-    target_info->swim_state = target_ms->swim_state;
+    *target_id = (swim_member_id_t)target_ms->id;
+    *target_inc_nr = target_ms->swim_state.inc_nr;
+    *target_addr = target_ms->addr;
+
+    return 0;
+}
+
+static int ssg_get_swim_iping_targets(
+    void *group_data,
+    swim_member_id_t *target_id,
+    hg_addr_t *target_addrs)
+{
+    ssg_group_t *g = (ssg_group_t *)group_data;
+
+    assert(g != NULL);
 
     return 0;
 }
@@ -165,6 +194,44 @@ static void ssg_gen_rand_member_list(ssg_group_t *g)
     return;
 }
 
+static void ssg_get_swim_member_addr(
+    void *group_data,
+    swim_member_id_t id,
+    hg_addr_t *addr)
+{
+    ssg_group_t *g = (ssg_group_t *)group_data;
+    ssg_member_id_t ssg_id = (ssg_member_id_t)id;
+    ssg_member_state_t *ms;
+
+    assert(g != NULL);
+
+    HASH_FIND(hh, g->view.member_map, &ssg_id, sizeof(ssg_member_id_t), ms);
+    /* XXX ASSERT */
+
+    *addr = ms->addr;
+
+    return;
+}
+
+static void ssg_get_swim_member_state(
+    void *group_data,
+    swim_member_id_t id,
+    swim_member_state_t *state)
+{
+    ssg_group_t *g = (ssg_group_t *)group_data;
+    ssg_member_id_t ssg_id = (ssg_member_id_t)id;
+    ssg_member_state_t *ms;
+
+    assert(g != NULL);
+
+    HASH_FIND(hh, g->view.member_map, &ssg_id, sizeof(ssg_member_id_t), ms);
+    /* XXX ASSERT */
+
+    *state = ms->swim_state;
+
+    return;
+}
+
 ssg_group_id_t ssg_group_create(
     const char * group_name,
     const char * const group_addr_strs[],
@@ -232,6 +299,9 @@ ssg_group_id_t ssg_group_create(
     // due to timing skew of different ranks initializing swim
     swim_group_mgmt_callbacks_t swim_callbacks = {
         .get_dping_target = &ssg_get_swim_dping_target,
+        .get_iping_targets = &ssg_get_swim_iping_targets,
+        .get_member_addr = ssg_get_swim_member_addr,
+        .get_member_state = ssg_get_swim_member_state,
     };
     g->swim_ctx = swim_init(ssg_inst->mid, g, (swim_member_id_t)g->self_id,
         swim_callbacks, 1);

src/swim-fd/swim-fd-internal.h

@@ -10,6 +10,7 @@
 #include <margo.h>
 
 #include "swim-fd.h"
+#include "utlist.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -42,14 +43,19 @@ struct swim_context
     margo_instance_id mid;
     /* void pointer to user group data */
     void *group_data;
+    /* XXX group mgmt callbacks */
+    swim_group_mgmt_callbacks_t swim_callbacks;
     /* XXX other state */
     swim_member_id_t self_id;
     swim_member_inc_nr_t self_inc_nr;
-    swim_dping_target_info_t dping_target_info;
-    int dping_target_acked;
+    swim_member_id_t dping_target_id;
+    swim_member_inc_nr_t dping_target_inc_nr;
+    hg_addr_t dping_target_addr;
     double dping_timeout;
-    /* XXX group mgmt callbacks */
-    swim_group_mgmt_callbacks_t swim_callbacks;
+    swim_member_id_t iping_target_ids[SWIM_MAX_SUBGROUP_SIZE];
+    hg_addr_t iping_target_addrs[SWIM_MAX_SUBGROUP_SIZE];
+    int iping_target_ndx;
+    int ping_target_acked;
     /* argobots pool for launching SWIM threads */
     ABT_pool swim_pool;
     /* swim protocol ULT handle */
@@ -58,36 +64,31 @@ struct swim_context
     double prot_period_len;
     int prot_susp_timeout;
     int prot_subgroup_sz;
-    /* SWIM internal state */
-    int shutdown_flag;
-    hg_addr_t iping_subgroup_addrs[SWIM_MAX_SUBGROUP_SIZE];
-#if 0
     /* current membership state */
     void *suspect_list;
+#if 0
     void *recent_update_list;
 #endif
+    /* XXX */
+    int shutdown_flag;
 };
 
-#if 0
-typedef struct swim_member_update swim_member_update_t;
-
-struct swim_member_update
+typedef struct swim_member_update
 {
-    ssg_member_id_t id;
+    swim_member_id_t id;
     swim_member_status_t status;
     swim_member_inc_nr_t inc_nr;
-};
-#endif
+} swim_member_update_t;
 
 /* SWIM ping function prototypes */
 void swim_register_ping_rpcs(
     swim_context_t * swim_ctx);
 void swim_dping_send_ult(
     void * t_arg);
-#if 0
 void swim_iping_send_ult(
     void * t_arg);
 
+#if 0
 /* SWIM membership update function prototypes */
 void swim_retrieve_membership_updates(
     ssg_group_t * g,

src/swim-fd/swim-fd.c

@@ -14,13 +14,14 @@
 #include "swim-fd.h"
 #include "swim-fd-internal.h"
 
-#if 0
 typedef struct swim_suspect_member_link
 {
+    swim_member_id_t member_id;
     double susp_start;
     struct swim_suspect_member_link *next;
 } swim_suspect_member_link_t;
 
+#if 0
 typedef struct swim_member_update_link
 {
     swim_member_update_t update;
@@ -35,10 +36,11 @@ static void swim_prot_ult(
 static void swim_tick_ult(
     void *t_arg);
 
-#if 0
 /* SWIM group membership utility function prototypes */
 static void swim_suspect_member(
-    ssg_group_t *g, ssg_member_id_t member_id, swim_member_inc_nr_t inc_nr);
+    swim_context_t *swim_ctx, swim_member_id_t member_id,
+    swim_member_inc_nr_t inc_nr);
+#if 0
 static void swim_unsuspect_member(
     ssg_group_t *g, ssg_member_id_t member_id, swim_member_inc_nr_t inc_nr);
 static void swim_kill_member(
@@ -61,7 +63,7 @@ swim_context_t * swim_init(
     int active)
 {
     swim_context_t *swim_ctx;
-    int i, ret;
+    int ret;
 
     /* allocate structure for storing swim context */
     swim_ctx = malloc(sizeof(*swim_ctx));
@@ -72,17 +74,18 @@ swim_context_t * swim_init(
     swim_ctx->self_id = self_id;
     swim_ctx->self_inc_nr = 0;
     swim_ctx->swim_callbacks = swim_callbacks;
-
-    /* initialize SWIM context */
     margo_get_handler_pool(swim_ctx->mid, &swim_ctx->swim_pool);
-    for(i = 0; i < SWIM_MAX_SUBGROUP_SIZE; i++)
-        swim_ctx->iping_subgroup_addrs[i] = HG_ADDR_NULL;
 
     /* set protocol parameters */
     swim_ctx->prot_period_len = SWIM_DEF_PROTOCOL_PERIOD_LEN;
     swim_ctx->prot_susp_timeout = SWIM_DEF_SUSPECT_TIMEOUT;
     swim_ctx->prot_subgroup_sz = SWIM_DEF_SUBGROUP_SIZE;
 
+    /* NOTE: set this flag so we don't inadvertently suspect a member
+     * on the first iteration of the protocol
+     */
+    swim_ctx->ping_target_acked = 1;
+
     swim_register_ping_rpcs(swim_ctx);
 
     if(active)
@@ -149,18 +152,18 @@ static void swim_tick_ult(
     /* check whether the ping target from the previous protocol tick
      * ever successfully acked a (direct/indirect) ping request
      */
-    if((swim_ctx->ping_target != SSG_MEMBER_ID_INVALID) &&
-        !(swim_ctx->ping_target_acked))
+    if(!(swim_ctx->ping_target_acked))
     {
         /* no response from direct/indirect pings, suspect this member */
-        swim_suspect_member(g, swim_ctx->ping_target, swim_ctx->ping_target_inc_nr);
+        swim_suspect_member(swim_ctx, swim_ctx->dping_target_id,
+            swim_ctx->dping_target_inc_nr);
     }
 #endif
 
-    /* pick a random member from view and ping */
+    /* pick a random member from view to ping */
     ret = swim_ctx->swim_callbacks.get_dping_target(
-            swim_ctx->group_data,
-            &swim_ctx->dping_target_info);
+        swim_ctx->group_data, &swim_ctx->dping_target_id,
+        &swim_ctx->dping_target_inc_nr, &swim_ctx->dping_target_addr);
     if(ret != 0)
     {
         /* no available members, back out */
@@ -172,7 +175,7 @@ static void swim_tick_ult(
     swim_ctx->dping_timeout = 250.0;
 
     /* kick off dping request ULT */
-    swim_ctx->dping_target_acked = 0;
+    swim_ctx->ping_target_acked = 0;
     ret = ABT_thread_create(swim_ctx->swim_pool, swim_dping_send_ult, swim_ctx,
         ABT_THREAD_ATTR_NULL, NULL);
     if(ret != ABT_SUCCESS)
@@ -184,27 +187,26 @@ static void swim_tick_ult(
     /* sleep for an RTT and wait for an ack for this dping req */
     margo_thread_sleep(swim_ctx->mid, swim_ctx->dping_timeout);
 
-#if 0
     /* if we don't hear back from the target after an RTT, kick off
      * a set of indirect pings to a subgroup of group members
      */
     if(!(swim_ctx->ping_target_acked) && (swim_ctx->prot_subgroup_sz > 0))
     {
         /* get a random subgroup of members to send indirect pings to */
-        int this_subgroup_sz = swim_get_rand_group_member_set(g,
-            swim_ctx->subgroup_members, swim_ctx->prot_subgroup_sz,
-            swim_ctx->ping_target);
-        if(this_subgroup_sz == 0)
+        int iping_target_count = swim_ctx->swim_callbacks.get_iping_targets(
+            swim_ctx->group_data, swim_ctx->iping_target_ids, swim_ctx->iping_target_addrs);
+        if(iping_target_count == 0)
         {
             /* no available subgroup members, back out */
-            SSG_DEBUG(g, "SWIM: no subgroup members available to iping\n");
+            SWIM_DEBUG(swim_ctx, "no subgroup members available to iping\n");
             return;
         }
 
-        for(i = 0; i < this_subgroup_sz; i++)
+        swim_ctx->iping_target_ndx = 0;
+        for(i = 0; i < iping_target_count; i++)
         {
-            ret = ABT_thread_create(swim_ctx->prot_pool, swim_iping_send_ult, g,
-                ABT_THREAD_ATTR_NULL, NULL);
+            ret = ABT_thread_create(swim_ctx->swim_pool, swim_iping_send_ult,
+                swim_ctx, ABT_THREAD_ATTR_NULL, NULL);
             if(ret != ABT_SUCCESS)
             {
                 fprintf(stderr, "Error: unable to create ULT for SWIM iping send\n");
@@ -212,7 +214,6 @@ static void swim_tick_ult(
             }
         }
     }
-#endif
 
     return;
 }
@@ -345,33 +346,48 @@ void swim_apply_membership_updates(
 
     return;
 }
+#endif
 
 /*******************************************
  * SWIM group membership utility functions *
  *******************************************/
 
+#if 0
 static void swim_suspect_member(
-    ssg_group_t *g, ssg_member_id_t member_id, swim_member_inc_nr_t inc_nr)
+    swim_context_t *swim_ctx, swim_member_id_t member_id, swim_member_inc_nr_t inc_nr)
 {
-    swim_context_t *swim_ctx = g->swim_ctx;
+    swim_member_state_t *cur_swim_state;
     swim_suspect_member_link_t *iter, *tmp;
     swim_suspect_member_link_t *suspect_link = NULL;
     swim_suspect_member_link_t *suspect_list_p =
         (swim_suspect_member_link_t *)swim_ctx->suspect_list;
     swim_member_update_t update;
 
-    /* ignore updates for dead members */
+    /* if there is no suspicion timeout, just kill the member */
+    if(swim_ctx->prot_susp_timeout == 0)
+    {
 #if 0
-    if(!(g->view.member_states[member_id].is_member))
-        return;
+        swim_kill_member(g, member_id, inc_nr);
 #endif
+        return;
+    }
+
+    /* XXX MUTEX */
+
+    /* get current swim state for member */
+    swim_ctx->swim_callbacks.get_member_state(
+        swim_ctx, member_id, &cur_swim_state);
+
+    /* ignore updates for dead members */
+    if(cur_swim_state.status == SWIM_MEMBER_DEAD)
+        return;
 
     /* determine if this member is already suspected */
     LL_FOREACH_SAFE(suspect_list_p, iter, tmp)
     {
         if(iter->member_id == member_id)
         {
-            if(inc_nr <= swim_ctx->member_inc_nrs[member_id])
+            if(inc_nr <= cur_swim_state.inc_nr)
             {
                 /* ignore a suspicion in an incarnation number less than
                  * or equal to the current suspicion's incarnation
@@ -388,18 +404,10 @@ static void swim_suspect_member(
     }
 
     /* ignore suspicions for a member that is alive in a newer incarnation */
-    if((suspect_link == NULL) && (inc_nr < swim_ctx->member_inc_nrs[member_id]))
-        return;
-
-    /* if there is no suspicion timeout, just kill the member */
-    if(swim_ctx->prot_susp_timeout == 0)
-    {
-        swim_kill_member(g, member_id, inc_nr);
+    if((suspect_link == NULL) && (inc_nr < cur_swim_state.inc_nr))
         return;
-    }
 
-    SSG_DEBUG(g, "SWIM: member %d SUSPECT (inc_nr=%d)\n",
-        (int)member_id, (int)inc_nr);
+    SWIM_DEBUG(swim_ctx, "member %lu SUSPECT (inc_nr=%lu)\n", member_id, inc_nr);
 
     if(suspect_link == NULL)
     {
@@ -417,9 +425,10 @@ static void swim_suspect_member(
     /* add to end of suspect list */
     LL_APPEND(suspect_list_p, suspect_link);
 
-    /* update swim membership state */
+    /* XXX XXX XXX XXX update swim membership state */
     swim_ctx->member_inc_nrs[member_id] = inc_nr;
 
+#if 0
     /* add this update to recent update list so it will be piggybacked
      * on future protocol messages
      */
@@ -427,6 +436,7 @@ static void swim_suspect_member(
     update.status = SWIM_MEMBER_SUSPECT;
     update.inc_nr = inc_nr;
     swim_add_recent_member_update(g, update);
+#endif
 
     return;
 }

src/swim-fd/swim-fd.h

@@ -32,13 +32,6 @@ typedef struct swim_member_state
     swim_member_status_t status;
 } swim_member_state_t;
 
-typedef struct swim_dping_target_info
-{
-    swim_member_id_t id;
-    hg_addr_t addr;
-    swim_member_state_t swim_state;
-} swim_dping_target_info_t;
-
 #define SWIM_MEMBER_STATE_INIT(__ms) do { \
     __ms.inc_nr = 0; \
     __ms.status = SWIM_MEMBER_ALIVE; \
@@ -47,11 +40,28 @@ typedef struct swim_dping_target_info
 /* XXX rename once more clear what all is here */
 typedef struct swim_group_mgmt_callbacks
 {
+    /* XXX RET VALS */
     int (*get_dping_target)(
             void *group_data,
-            swim_dping_target_info_t *target_info
+            swim_member_id_t *target_id,
+            swim_member_inc_nr_t *inc_nr,
+            hg_addr_t *target_addr
+            );
+    int (*get_iping_targets)(
+            void *group_data,
+            swim_member_id_t *target_ids,
+            hg_addr_t *target_addrs
+            );
+    void (*get_member_addr)(
+            void *group_data,
+            swim_member_id_t id,
+            hg_addr_t *addr
+            );
+    void (*get_member_state)(
+            void *group_data,
+            swim_member_id_t id,
+            swim_member_state_t **state
             );
-    /* get_rand_iping_subgroup */
 } swim_group_mgmt_callbacks_t;
 
 /* Initialize SWIM */