Commit 66416549 authored by Shane Snyder's avatar Shane Snyder

more swim updates

parent 03c800f0
...@@ -125,13 +125,30 @@ int ssg_finalize() ...@@ -125,13 +125,30 @@ int ssg_finalize()
static int ssg_get_swim_dping_target( static int ssg_get_swim_dping_target(
void *group_data, void *group_data,
swim_dping_target_info_t *target_info); swim_member_id_t *target_id,
swim_member_inc_nr_t *target_inc_nr,
hg_addr_t *target_addr);
static int ssg_get_swim_iping_targets(
void *group_data,
swim_member_id_t *target_ids,
hg_addr_t *target_addrs);
static void ssg_get_swim_member_addr(
void *group_data,
swim_member_id_t id,
hg_addr_t *target_addr);
static void ssg_get_swim_member_state(
void *group_data,
swim_member_id_t id,
swim_member_state_t *state);
static void ssg_gen_rand_member_list( static void ssg_gen_rand_member_list(
ssg_group_t *g); ssg_group_t *g);
static int ssg_get_swim_dping_target( static int ssg_get_swim_dping_target(
void *group_data, void *group_data,
swim_dping_target_info_t *target_info) swim_member_id_t *target_id,
swim_member_inc_nr_t *target_inc_nr,
hg_addr_t *target_addr)
{ {
ssg_group_t *g = (ssg_group_t *)group_data; ssg_group_t *g = (ssg_group_t *)group_data;
ssg_member_state_t *target_ms; ssg_member_state_t *target_ms;
...@@ -145,9 +162,21 @@ static int ssg_get_swim_dping_target( ...@@ -145,9 +162,21 @@ static int ssg_get_swim_dping_target(
/* pull random member off head of list and return addr */ /* pull random member off head of list and return addr */
target_ms = g->member_list; target_ms = g->member_list;
LL_DELETE(g->member_list, target_ms); LL_DELETE(g->member_list, target_ms);
target_info->id = (swim_member_id_t)target_ms->id; *target_id = (swim_member_id_t)target_ms->id;
target_info->addr = target_ms->addr; *target_inc_nr = target_ms->swim_state.inc_nr;
target_info->swim_state = target_ms->swim_state; *target_addr = target_ms->addr;
return 0;
}
static int ssg_get_swim_iping_targets(
void *group_data,
swim_member_id_t *target_id,
hg_addr_t *target_addrs)
{
ssg_group_t *g = (ssg_group_t *)group_data;
assert(g != NULL);
return 0; return 0;
} }
...@@ -165,6 +194,44 @@ static void ssg_gen_rand_member_list(ssg_group_t *g) ...@@ -165,6 +194,44 @@ static void ssg_gen_rand_member_list(ssg_group_t *g)
return; return;
} }
static void ssg_get_swim_member_addr(
void *group_data,
swim_member_id_t id,
hg_addr_t *addr)
{
ssg_group_t *g = (ssg_group_t *)group_data;
ssg_member_id_t ssg_id = (ssg_member_id_t)id;
ssg_member_state_t *ms;
assert(g != NULL);
HASH_FIND(hh, g->view.member_map, &ssg_id, sizeof(ssg_member_id_t), ms);
/* XXX ASSERT */
*addr = ms->addr;
return;
}
static void ssg_get_swim_member_state(
void *group_data,
swim_member_id_t id,
swim_member_state_t *state)
{
ssg_group_t *g = (ssg_group_t *)group_data;
ssg_member_id_t ssg_id = (ssg_member_id_t)id;
ssg_member_state_t *ms;
assert(g != NULL);
HASH_FIND(hh, g->view.member_map, &ssg_id, sizeof(ssg_member_id_t), ms);
/* XXX ASSERT */
*state = ms->swim_state;
return;
}
ssg_group_id_t ssg_group_create( ssg_group_id_t ssg_group_create(
const char * group_name, const char * group_name,
const char * const group_addr_strs[], const char * const group_addr_strs[],
...@@ -232,6 +299,9 @@ ssg_group_id_t ssg_group_create( ...@@ -232,6 +299,9 @@ ssg_group_id_t ssg_group_create(
// due to timing skew of different ranks initializing swim // due to timing skew of different ranks initializing swim
swim_group_mgmt_callbacks_t swim_callbacks = { swim_group_mgmt_callbacks_t swim_callbacks = {
.get_dping_target = &ssg_get_swim_dping_target, .get_dping_target = &ssg_get_swim_dping_target,
.get_iping_targets = &ssg_get_swim_iping_targets,
.get_member_addr = ssg_get_swim_member_addr,
.get_member_state = ssg_get_swim_member_state,
}; };
g->swim_ctx = swim_init(ssg_inst->mid, g, (swim_member_id_t)g->self_id, g->swim_ctx = swim_init(ssg_inst->mid, g, (swim_member_id_t)g->self_id,
swim_callbacks, 1); swim_callbacks, 1);
......
...@@ -10,6 +10,7 @@ ...@@ -10,6 +10,7 @@
#include <margo.h> #include <margo.h>
#include "swim-fd.h" #include "swim-fd.h"
#include "utlist.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
...@@ -42,14 +43,19 @@ struct swim_context ...@@ -42,14 +43,19 @@ struct swim_context
margo_instance_id mid; margo_instance_id mid;
/* void pointer to user group data */ /* void pointer to user group data */
void *group_data; void *group_data;
/* XXX group mgmt callbacks */
swim_group_mgmt_callbacks_t swim_callbacks;
/* XXX other state */ /* XXX other state */
swim_member_id_t self_id; swim_member_id_t self_id;
swim_member_inc_nr_t self_inc_nr; swim_member_inc_nr_t self_inc_nr;
swim_dping_target_info_t dping_target_info; swim_member_id_t dping_target_id;
int dping_target_acked; swim_member_inc_nr_t dping_target_inc_nr;
hg_addr_t dping_target_addr;
double dping_timeout; double dping_timeout;
/* XXX group mgmt callbacks */ swim_member_id_t iping_target_ids[SWIM_MAX_SUBGROUP_SIZE];
swim_group_mgmt_callbacks_t swim_callbacks; hg_addr_t iping_target_addrs[SWIM_MAX_SUBGROUP_SIZE];
int iping_target_ndx;
int ping_target_acked;
/* argobots pool for launching SWIM threads */ /* argobots pool for launching SWIM threads */
ABT_pool swim_pool; ABT_pool swim_pool;
/* swim protocol ULT handle */ /* swim protocol ULT handle */
...@@ -58,36 +64,31 @@ struct swim_context ...@@ -58,36 +64,31 @@ struct swim_context
double prot_period_len; double prot_period_len;
int prot_susp_timeout; int prot_susp_timeout;
int prot_subgroup_sz; int prot_subgroup_sz;
/* SWIM internal state */
int shutdown_flag;
hg_addr_t iping_subgroup_addrs[SWIM_MAX_SUBGROUP_SIZE];
#if 0
/* current membership state */ /* current membership state */
void *suspect_list; void *suspect_list;
#if 0
void *recent_update_list; void *recent_update_list;
#endif #endif
/* XXX */
int shutdown_flag;
}; };
#if 0 typedef struct swim_member_update
typedef struct swim_member_update swim_member_update_t;
struct swim_member_update
{ {
ssg_member_id_t id; swim_member_id_t id;
swim_member_status_t status; swim_member_status_t status;
swim_member_inc_nr_t inc_nr; swim_member_inc_nr_t inc_nr;
}; } swim_member_update_t;
#endif
/* SWIM ping function prototypes */ /* SWIM ping function prototypes */
void swim_register_ping_rpcs( void swim_register_ping_rpcs(
swim_context_t * swim_ctx); swim_context_t * swim_ctx);
void swim_dping_send_ult( void swim_dping_send_ult(
void * t_arg); void * t_arg);
#if 0
void swim_iping_send_ult( void swim_iping_send_ult(
void * t_arg); void * t_arg);
#if 0
/* SWIM membership update function prototypes */ /* SWIM membership update function prototypes */
void swim_retrieve_membership_updates( void swim_retrieve_membership_updates(
ssg_group_t * g, ssg_group_t * g,
......
This diff is collapsed.
...@@ -14,13 +14,14 @@ ...@@ -14,13 +14,14 @@
#include "swim-fd.h" #include "swim-fd.h"
#include "swim-fd-internal.h" #include "swim-fd-internal.h"
#if 0
typedef struct swim_suspect_member_link typedef struct swim_suspect_member_link
{ {
swim_member_id_t member_id;
double susp_start; double susp_start;
struct swim_suspect_member_link *next; struct swim_suspect_member_link *next;
} swim_suspect_member_link_t; } swim_suspect_member_link_t;
#if 0
typedef struct swim_member_update_link typedef struct swim_member_update_link
{ {
swim_member_update_t update; swim_member_update_t update;
...@@ -35,10 +36,11 @@ static void swim_prot_ult( ...@@ -35,10 +36,11 @@ static void swim_prot_ult(
static void swim_tick_ult( static void swim_tick_ult(
void *t_arg); void *t_arg);
#if 0
/* SWIM group membership utility function prototypes */ /* SWIM group membership utility function prototypes */
static void swim_suspect_member( static void swim_suspect_member(
ssg_group_t *g, ssg_member_id_t member_id, swim_member_inc_nr_t inc_nr); swim_context_t *swim_ctx, swim_member_id_t member_id,
swim_member_inc_nr_t inc_nr);
#if 0
static void swim_unsuspect_member( static void swim_unsuspect_member(
ssg_group_t *g, ssg_member_id_t member_id, swim_member_inc_nr_t inc_nr); ssg_group_t *g, ssg_member_id_t member_id, swim_member_inc_nr_t inc_nr);
static void swim_kill_member( static void swim_kill_member(
...@@ -61,7 +63,7 @@ swim_context_t * swim_init( ...@@ -61,7 +63,7 @@ swim_context_t * swim_init(
int active) int active)
{ {
swim_context_t *swim_ctx; swim_context_t *swim_ctx;
int i, ret; int ret;
/* allocate structure for storing swim context */ /* allocate structure for storing swim context */
swim_ctx = malloc(sizeof(*swim_ctx)); swim_ctx = malloc(sizeof(*swim_ctx));
...@@ -72,17 +74,18 @@ swim_context_t * swim_init( ...@@ -72,17 +74,18 @@ swim_context_t * swim_init(
swim_ctx->self_id = self_id; swim_ctx->self_id = self_id;
swim_ctx->self_inc_nr = 0; swim_ctx->self_inc_nr = 0;
swim_ctx->swim_callbacks = swim_callbacks; swim_ctx->swim_callbacks = swim_callbacks;
/* initialize SWIM context */
margo_get_handler_pool(swim_ctx->mid, &swim_ctx->swim_pool); margo_get_handler_pool(swim_ctx->mid, &swim_ctx->swim_pool);
for(i = 0; i < SWIM_MAX_SUBGROUP_SIZE; i++)
swim_ctx->iping_subgroup_addrs[i] = HG_ADDR_NULL;
/* set protocol parameters */ /* set protocol parameters */
swim_ctx->prot_period_len = SWIM_DEF_PROTOCOL_PERIOD_LEN; swim_ctx->prot_period_len = SWIM_DEF_PROTOCOL_PERIOD_LEN;
swim_ctx->prot_susp_timeout = SWIM_DEF_SUSPECT_TIMEOUT; swim_ctx->prot_susp_timeout = SWIM_DEF_SUSPECT_TIMEOUT;
swim_ctx->prot_subgroup_sz = SWIM_DEF_SUBGROUP_SIZE; swim_ctx->prot_subgroup_sz = SWIM_DEF_SUBGROUP_SIZE;
/* NOTE: set this flag so we don't inadvertently suspect a member
* on the first iteration of the protocol
*/
swim_ctx->ping_target_acked = 1;
swim_register_ping_rpcs(swim_ctx); swim_register_ping_rpcs(swim_ctx);
if(active) if(active)
...@@ -149,18 +152,18 @@ static void swim_tick_ult( ...@@ -149,18 +152,18 @@ static void swim_tick_ult(
/* check whether the ping target from the previous protocol tick /* check whether the ping target from the previous protocol tick
* ever successfully acked a (direct/indirect) ping request * ever successfully acked a (direct/indirect) ping request
*/ */
if((swim_ctx->ping_target != SSG_MEMBER_ID_INVALID) && if(!(swim_ctx->ping_target_acked))
!(swim_ctx->ping_target_acked))
{ {
/* no response from direct/indirect pings, suspect this member */ /* no response from direct/indirect pings, suspect this member */
swim_suspect_member(g, swim_ctx->ping_target, swim_ctx->ping_target_inc_nr); swim_suspect_member(swim_ctx, swim_ctx->dping_target_id,
swim_ctx->dping_target_inc_nr);
} }
#endif #endif
/* pick a random member from view and ping */ /* pick a random member from view to ping */
ret = swim_ctx->swim_callbacks.get_dping_target( ret = swim_ctx->swim_callbacks.get_dping_target(
swim_ctx->group_data, swim_ctx->group_data, &swim_ctx->dping_target_id,
&swim_ctx->dping_target_info); &swim_ctx->dping_target_inc_nr, &swim_ctx->dping_target_addr);
if(ret != 0) if(ret != 0)
{ {
/* no available members, back out */ /* no available members, back out */
...@@ -172,7 +175,7 @@ static void swim_tick_ult( ...@@ -172,7 +175,7 @@ static void swim_tick_ult(
swim_ctx->dping_timeout = 250.0; swim_ctx->dping_timeout = 250.0;
/* kick off dping request ULT */ /* kick off dping request ULT */
swim_ctx->dping_target_acked = 0; swim_ctx->ping_target_acked = 0;
ret = ABT_thread_create(swim_ctx->swim_pool, swim_dping_send_ult, swim_ctx, ret = ABT_thread_create(swim_ctx->swim_pool, swim_dping_send_ult, swim_ctx,
ABT_THREAD_ATTR_NULL, NULL); ABT_THREAD_ATTR_NULL, NULL);
if(ret != ABT_SUCCESS) if(ret != ABT_SUCCESS)
...@@ -184,27 +187,26 @@ static void swim_tick_ult( ...@@ -184,27 +187,26 @@ static void swim_tick_ult(
/* sleep for an RTT and wait for an ack for this dping req */ /* sleep for an RTT and wait for an ack for this dping req */
margo_thread_sleep(swim_ctx->mid, swim_ctx->dping_timeout); margo_thread_sleep(swim_ctx->mid, swim_ctx->dping_timeout);
#if 0
/* if we don't hear back from the target after an RTT, kick off /* if we don't hear back from the target after an RTT, kick off
* a set of indirect pings to a subgroup of group members * a set of indirect pings to a subgroup of group members
*/ */
if(!(swim_ctx->ping_target_acked) && (swim_ctx->prot_subgroup_sz > 0)) if(!(swim_ctx->ping_target_acked) && (swim_ctx->prot_subgroup_sz > 0))
{ {
/* get a random subgroup of members to send indirect pings to */ /* get a random subgroup of members to send indirect pings to */
int this_subgroup_sz = swim_get_rand_group_member_set(g, int iping_target_count = swim_ctx->swim_callbacks.get_iping_targets(
swim_ctx->subgroup_members, swim_ctx->prot_subgroup_sz, swim_ctx->group_data, swim_ctx->iping_target_ids, swim_ctx->iping_target_addrs);
swim_ctx->ping_target); if(iping_target_count == 0)
if(this_subgroup_sz == 0)
{ {
/* no available subgroup members, back out */ /* no available subgroup members, back out */
SSG_DEBUG(g, "SWIM: no subgroup members available to iping\n"); SWIM_DEBUG(swim_ctx, "no subgroup members available to iping\n");
return; return;
} }
for(i = 0; i < this_subgroup_sz; i++) swim_ctx->iping_target_ndx = 0;
for(i = 0; i < iping_target_count; i++)
{ {
ret = ABT_thread_create(swim_ctx->prot_pool, swim_iping_send_ult, g, ret = ABT_thread_create(swim_ctx->swim_pool, swim_iping_send_ult,
ABT_THREAD_ATTR_NULL, NULL); swim_ctx, ABT_THREAD_ATTR_NULL, NULL);
if(ret != ABT_SUCCESS) if(ret != ABT_SUCCESS)
{ {
fprintf(stderr, "Error: unable to create ULT for SWIM iping send\n"); fprintf(stderr, "Error: unable to create ULT for SWIM iping send\n");
...@@ -212,7 +214,6 @@ static void swim_tick_ult( ...@@ -212,7 +214,6 @@ static void swim_tick_ult(
} }
} }
} }
#endif
return; return;
} }
...@@ -345,33 +346,48 @@ void swim_apply_membership_updates( ...@@ -345,33 +346,48 @@ void swim_apply_membership_updates(
return; return;
} }
#endif
/******************************************* /*******************************************
* SWIM group membership utility functions * * SWIM group membership utility functions *
*******************************************/ *******************************************/
#if 0
static void swim_suspect_member( static void swim_suspect_member(
ssg_group_t *g, ssg_member_id_t member_id, swim_member_inc_nr_t inc_nr) swim_context_t *swim_ctx, swim_member_id_t member_id, swim_member_inc_nr_t inc_nr)
{ {
swim_context_t *swim_ctx = g->swim_ctx; swim_member_state_t *cur_swim_state;
swim_suspect_member_link_t *iter, *tmp; swim_suspect_member_link_t *iter, *tmp;
swim_suspect_member_link_t *suspect_link = NULL; swim_suspect_member_link_t *suspect_link = NULL;
swim_suspect_member_link_t *suspect_list_p = swim_suspect_member_link_t *suspect_list_p =
(swim_suspect_member_link_t *)swim_ctx->suspect_list; (swim_suspect_member_link_t *)swim_ctx->suspect_list;
swim_member_update_t update; swim_member_update_t update;
/* ignore updates for dead members */ /* if there is no suspicion timeout, just kill the member */
if(swim_ctx->prot_susp_timeout == 0)
{
#if 0 #if 0
if(!(g->view.member_states[member_id].is_member)) swim_kill_member(g, member_id, inc_nr);
return;
#endif #endif
return;
}
/* XXX MUTEX */
/* get current swim state for member */
swim_ctx->swim_callbacks.get_member_state(
swim_ctx, member_id, &cur_swim_state);
/* ignore updates for dead members */
if(cur_swim_state.status == SWIM_MEMBER_DEAD)
return;
/* determine if this member is already suspected */ /* determine if this member is already suspected */
LL_FOREACH_SAFE(suspect_list_p, iter, tmp) LL_FOREACH_SAFE(suspect_list_p, iter, tmp)
{ {
if(iter->member_id == member_id) if(iter->member_id == member_id)
{ {
if(inc_nr <= swim_ctx->member_inc_nrs[member_id]) if(inc_nr <= cur_swim_state.inc_nr)
{ {
/* ignore a suspicion in an incarnation number less than /* ignore a suspicion in an incarnation number less than
* or equal to the current suspicion's incarnation * or equal to the current suspicion's incarnation
...@@ -388,18 +404,10 @@ static void swim_suspect_member( ...@@ -388,18 +404,10 @@ static void swim_suspect_member(
} }
/* ignore suspicions for a member that is alive in a newer incarnation */ /* ignore suspicions for a member that is alive in a newer incarnation */
if((suspect_link == NULL) && (inc_nr < swim_ctx->member_inc_nrs[member_id])) if((suspect_link == NULL) && (inc_nr < cur_swim_state.inc_nr))
return;
/* if there is no suspicion timeout, just kill the member */
if(swim_ctx->prot_susp_timeout == 0)
{
swim_kill_member(g, member_id, inc_nr);
return; return;
}
SSG_DEBUG(g, "SWIM: member %d SUSPECT (inc_nr=%d)\n", SWIM_DEBUG(swim_ctx, "member %lu SUSPECT (inc_nr=%lu)\n", member_id, inc_nr);
(int)member_id, (int)inc_nr);
if(suspect_link == NULL) if(suspect_link == NULL)
{ {
...@@ -417,9 +425,10 @@ static void swim_suspect_member( ...@@ -417,9 +425,10 @@ static void swim_suspect_member(
/* add to end of suspect list */ /* add to end of suspect list */
LL_APPEND(suspect_list_p, suspect_link); LL_APPEND(suspect_list_p, suspect_link);
/* update swim membership state */ /* XXX XXX XXX XXX update swim membership state */
swim_ctx->member_inc_nrs[member_id] = inc_nr; swim_ctx->member_inc_nrs[member_id] = inc_nr;
#if 0
/* add this update to recent update list so it will be piggybacked /* add this update to recent update list so it will be piggybacked
* on future protocol messages * on future protocol messages
*/ */
...@@ -427,6 +436,7 @@ static void swim_suspect_member( ...@@ -427,6 +436,7 @@ static void swim_suspect_member(
update.status = SWIM_MEMBER_SUSPECT; update.status = SWIM_MEMBER_SUSPECT;
update.inc_nr = inc_nr; update.inc_nr = inc_nr;
swim_add_recent_member_update(g, update); swim_add_recent_member_update(g, update);
#endif
return; return;
} }
......
...@@ -32,13 +32,6 @@ typedef struct swim_member_state ...@@ -32,13 +32,6 @@ typedef struct swim_member_state
swim_member_status_t status; swim_member_status_t status;
} swim_member_state_t; } swim_member_state_t;
typedef struct swim_dping_target_info
{
swim_member_id_t id;
hg_addr_t addr;
swim_member_state_t swim_state;
} swim_dping_target_info_t;
#define SWIM_MEMBER_STATE_INIT(__ms) do { \ #define SWIM_MEMBER_STATE_INIT(__ms) do { \
__ms.inc_nr = 0; \ __ms.inc_nr = 0; \
__ms.status = SWIM_MEMBER_ALIVE; \ __ms.status = SWIM_MEMBER_ALIVE; \
...@@ -47,11 +40,28 @@ typedef struct swim_dping_target_info ...@@ -47,11 +40,28 @@ typedef struct swim_dping_target_info
/* XXX rename once more clear what all is here */ /* XXX rename once more clear what all is here */
typedef struct swim_group_mgmt_callbacks typedef struct swim_group_mgmt_callbacks
{ {
/* XXX RET VALS */
int (*get_dping_target)( int (*get_dping_target)(
void *group_data, void *group_data,
swim_dping_target_info_t *target_info swim_member_id_t *target_id,
swim_member_inc_nr_t *inc_nr,
hg_addr_t *target_addr
);
int (*get_iping_targets)(
void *group_data,
swim_member_id_t *target_ids,
hg_addr_t *target_addrs
);
void (*get_member_addr)(
void *group_data,
swim_member_id_t id,
hg_addr_t *addr
);
void (*get_member_state)(
void *group_data,
swim_member_id_t id,
swim_member_state_t **state
); );
/* get_rand_iping_subgroup */
} swim_group_mgmt_callbacks_t; } swim_group_mgmt_callbacks_t;
/* Initialize SWIM */ /* Initialize SWIM */
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment