Commit 66416549 authored by Shane Snyder's avatar Shane Snyder

more swim updates

parent 03c800f0
......@@ -125,13 +125,30 @@ int ssg_finalize()
static int ssg_get_swim_dping_target(
void *group_data,
swim_dping_target_info_t *target_info);
swim_member_id_t *target_id,
swim_member_inc_nr_t *target_inc_nr,
hg_addr_t *target_addr);
static int ssg_get_swim_iping_targets(
void *group_data,
swim_member_id_t *target_ids,
hg_addr_t *target_addrs);
static void ssg_get_swim_member_addr(
void *group_data,
swim_member_id_t id,
hg_addr_t *target_addr);
static void ssg_get_swim_member_state(
void *group_data,
swim_member_id_t id,
swim_member_state_t *state);
static void ssg_gen_rand_member_list(
ssg_group_t *g);
static int ssg_get_swim_dping_target(
void *group_data,
swim_dping_target_info_t *target_info)
swim_member_id_t *target_id,
swim_member_inc_nr_t *target_inc_nr,
hg_addr_t *target_addr)
{
ssg_group_t *g = (ssg_group_t *)group_data;
ssg_member_state_t *target_ms;
......@@ -145,9 +162,21 @@ static int ssg_get_swim_dping_target(
/* pull random member off head of list and return addr */
target_ms = g->member_list;
LL_DELETE(g->member_list, target_ms);
target_info->id = (swim_member_id_t)target_ms->id;
target_info->addr = target_ms->addr;
target_info->swim_state = target_ms->swim_state;
*target_id = (swim_member_id_t)target_ms->id;
*target_inc_nr = target_ms->swim_state.inc_nr;
*target_addr = target_ms->addr;
return 0;
}
static int ssg_get_swim_iping_targets(
void *group_data,
swim_member_id_t *target_id,
hg_addr_t *target_addrs)
{
ssg_group_t *g = (ssg_group_t *)group_data;
assert(g != NULL);
return 0;
}
......@@ -165,6 +194,44 @@ static void ssg_gen_rand_member_list(ssg_group_t *g)
return;
}
static void ssg_get_swim_member_addr(
void *group_data,
swim_member_id_t id,
hg_addr_t *addr)
{
ssg_group_t *g = (ssg_group_t *)group_data;
ssg_member_id_t ssg_id = (ssg_member_id_t)id;
ssg_member_state_t *ms;
assert(g != NULL);
HASH_FIND(hh, g->view.member_map, &ssg_id, sizeof(ssg_member_id_t), ms);
/* XXX ASSERT */
*addr = ms->addr;
return;
}
static void ssg_get_swim_member_state(
void *group_data,
swim_member_id_t id,
swim_member_state_t *state)
{
ssg_group_t *g = (ssg_group_t *)group_data;
ssg_member_id_t ssg_id = (ssg_member_id_t)id;
ssg_member_state_t *ms;
assert(g != NULL);
HASH_FIND(hh, g->view.member_map, &ssg_id, sizeof(ssg_member_id_t), ms);
/* XXX ASSERT */
*state = ms->swim_state;
return;
}
ssg_group_id_t ssg_group_create(
const char * group_name,
const char * const group_addr_strs[],
......@@ -232,6 +299,9 @@ ssg_group_id_t ssg_group_create(
// due to timing skew of different ranks initializing swim
swim_group_mgmt_callbacks_t swim_callbacks = {
.get_dping_target = &ssg_get_swim_dping_target,
.get_iping_targets = &ssg_get_swim_iping_targets,
.get_member_addr = ssg_get_swim_member_addr,
.get_member_state = ssg_get_swim_member_state,
};
g->swim_ctx = swim_init(ssg_inst->mid, g, (swim_member_id_t)g->self_id,
swim_callbacks, 1);
......
......@@ -10,6 +10,7 @@
#include <margo.h>
#include "swim-fd.h"
#include "utlist.h"
#ifdef __cplusplus
extern "C" {
......@@ -42,14 +43,19 @@ struct swim_context
margo_instance_id mid;
/* void pointer to user group data */
void *group_data;
/* XXX group mgmt callbacks */
swim_group_mgmt_callbacks_t swim_callbacks;
/* XXX other state */
swim_member_id_t self_id;
swim_member_inc_nr_t self_inc_nr;
swim_dping_target_info_t dping_target_info;
int dping_target_acked;
swim_member_id_t dping_target_id;
swim_member_inc_nr_t dping_target_inc_nr;
hg_addr_t dping_target_addr;
double dping_timeout;
/* XXX group mgmt callbacks */
swim_group_mgmt_callbacks_t swim_callbacks;
swim_member_id_t iping_target_ids[SWIM_MAX_SUBGROUP_SIZE];
hg_addr_t iping_target_addrs[SWIM_MAX_SUBGROUP_SIZE];
int iping_target_ndx;
int ping_target_acked;
/* argobots pool for launching SWIM threads */
ABT_pool swim_pool;
/* swim protocol ULT handle */
......@@ -58,36 +64,31 @@ struct swim_context
double prot_period_len;
int prot_susp_timeout;
int prot_subgroup_sz;
/* SWIM internal state */
int shutdown_flag;
hg_addr_t iping_subgroup_addrs[SWIM_MAX_SUBGROUP_SIZE];
#if 0
/* current membership state */
void *suspect_list;
#if 0
void *recent_update_list;
#endif
/* XXX */
int shutdown_flag;
};
#if 0
typedef struct swim_member_update swim_member_update_t;
struct swim_member_update
typedef struct swim_member_update
{
ssg_member_id_t id;
swim_member_id_t id;
swim_member_status_t status;
swim_member_inc_nr_t inc_nr;
};
#endif
} swim_member_update_t;
/* SWIM ping function prototypes */
void swim_register_ping_rpcs(
swim_context_t * swim_ctx);
void swim_dping_send_ult(
void * t_arg);
#if 0
void swim_iping_send_ult(
void * t_arg);
#if 0
/* SWIM membership update function prototypes */
void swim_retrieve_membership_updates(
ssg_group_t * g,
......
This diff is collapsed.
......@@ -14,13 +14,14 @@
#include "swim-fd.h"
#include "swim-fd-internal.h"
#if 0
typedef struct swim_suspect_member_link
{
swim_member_id_t member_id;
double susp_start;
struct swim_suspect_member_link *next;
} swim_suspect_member_link_t;
#if 0
typedef struct swim_member_update_link
{
swim_member_update_t update;
......@@ -35,10 +36,11 @@ static void swim_prot_ult(
static void swim_tick_ult(
void *t_arg);
#if 0
/* SWIM group membership utility function prototypes */
static void swim_suspect_member(
ssg_group_t *g, ssg_member_id_t member_id, swim_member_inc_nr_t inc_nr);
swim_context_t *swim_ctx, swim_member_id_t member_id,
swim_member_inc_nr_t inc_nr);
#if 0
static void swim_unsuspect_member(
ssg_group_t *g, ssg_member_id_t member_id, swim_member_inc_nr_t inc_nr);
static void swim_kill_member(
......@@ -61,7 +63,7 @@ swim_context_t * swim_init(
int active)
{
swim_context_t *swim_ctx;
int i, ret;
int ret;
/* allocate structure for storing swim context */
swim_ctx = malloc(sizeof(*swim_ctx));
......@@ -72,17 +74,18 @@ swim_context_t * swim_init(
swim_ctx->self_id = self_id;
swim_ctx->self_inc_nr = 0;
swim_ctx->swim_callbacks = swim_callbacks;
/* initialize SWIM context */
margo_get_handler_pool(swim_ctx->mid, &swim_ctx->swim_pool);
for(i = 0; i < SWIM_MAX_SUBGROUP_SIZE; i++)
swim_ctx->iping_subgroup_addrs[i] = HG_ADDR_NULL;
/* set protocol parameters */
swim_ctx->prot_period_len = SWIM_DEF_PROTOCOL_PERIOD_LEN;
swim_ctx->prot_susp_timeout = SWIM_DEF_SUSPECT_TIMEOUT;
swim_ctx->prot_subgroup_sz = SWIM_DEF_SUBGROUP_SIZE;
/* NOTE: set this flag so we don't inadvertently suspect a member
* on the first iteration of the protocol
*/
swim_ctx->ping_target_acked = 1;
swim_register_ping_rpcs(swim_ctx);
if(active)
......@@ -149,18 +152,18 @@ static void swim_tick_ult(
/* check whether the ping target from the previous protocol tick
* ever successfully acked a (direct/indirect) ping request
*/
if((swim_ctx->ping_target != SSG_MEMBER_ID_INVALID) &&
!(swim_ctx->ping_target_acked))
if(!(swim_ctx->ping_target_acked))
{
/* no response from direct/indirect pings, suspect this member */
swim_suspect_member(g, swim_ctx->ping_target, swim_ctx->ping_target_inc_nr);
swim_suspect_member(swim_ctx, swim_ctx->dping_target_id,
swim_ctx->dping_target_inc_nr);
}
#endif
/* pick a random member from view and ping */
/* pick a random member from view to ping */
ret = swim_ctx->swim_callbacks.get_dping_target(
swim_ctx->group_data,
&swim_ctx->dping_target_info);
swim_ctx->group_data, &swim_ctx->dping_target_id,
&swim_ctx->dping_target_inc_nr, &swim_ctx->dping_target_addr);
if(ret != 0)
{
/* no available members, back out */
......@@ -172,7 +175,7 @@ static void swim_tick_ult(
swim_ctx->dping_timeout = 250.0;
/* kick off dping request ULT */
swim_ctx->dping_target_acked = 0;
swim_ctx->ping_target_acked = 0;
ret = ABT_thread_create(swim_ctx->swim_pool, swim_dping_send_ult, swim_ctx,
ABT_THREAD_ATTR_NULL, NULL);
if(ret != ABT_SUCCESS)
......@@ -184,27 +187,26 @@ static void swim_tick_ult(
/* sleep for an RTT and wait for an ack for this dping req */
margo_thread_sleep(swim_ctx->mid, swim_ctx->dping_timeout);
#if 0
/* if we don't hear back from the target after an RTT, kick off
* a set of indirect pings to a subgroup of group members
*/
if(!(swim_ctx->ping_target_acked) && (swim_ctx->prot_subgroup_sz > 0))
{
/* get a random subgroup of members to send indirect pings to */
int this_subgroup_sz = swim_get_rand_group_member_set(g,
swim_ctx->subgroup_members, swim_ctx->prot_subgroup_sz,
swim_ctx->ping_target);
if(this_subgroup_sz == 0)
int iping_target_count = swim_ctx->swim_callbacks.get_iping_targets(
swim_ctx->group_data, swim_ctx->iping_target_ids, swim_ctx->iping_target_addrs);
if(iping_target_count == 0)
{
/* no available subgroup members, back out */
SSG_DEBUG(g, "SWIM: no subgroup members available to iping\n");
SWIM_DEBUG(swim_ctx, "no subgroup members available to iping\n");
return;
}
for(i = 0; i < this_subgroup_sz; i++)
swim_ctx->iping_target_ndx = 0;
for(i = 0; i < iping_target_count; i++)
{
ret = ABT_thread_create(swim_ctx->prot_pool, swim_iping_send_ult, g,
ABT_THREAD_ATTR_NULL, NULL);
ret = ABT_thread_create(swim_ctx->swim_pool, swim_iping_send_ult,
swim_ctx, ABT_THREAD_ATTR_NULL, NULL);
if(ret != ABT_SUCCESS)
{
fprintf(stderr, "Error: unable to create ULT for SWIM iping send\n");
......@@ -212,7 +214,6 @@ static void swim_tick_ult(
}
}
}
#endif
return;
}
......@@ -345,33 +346,48 @@ void swim_apply_membership_updates(
return;
}
#endif
/*******************************************
* SWIM group membership utility functions *
*******************************************/
#if 0
static void swim_suspect_member(
ssg_group_t *g, ssg_member_id_t member_id, swim_member_inc_nr_t inc_nr)
swim_context_t *swim_ctx, swim_member_id_t member_id, swim_member_inc_nr_t inc_nr)
{
swim_context_t *swim_ctx = g->swim_ctx;
swim_member_state_t *cur_swim_state;
swim_suspect_member_link_t *iter, *tmp;
swim_suspect_member_link_t *suspect_link = NULL;
swim_suspect_member_link_t *suspect_list_p =
(swim_suspect_member_link_t *)swim_ctx->suspect_list;
swim_member_update_t update;
/* ignore updates for dead members */
/* if there is no suspicion timeout, just kill the member */
if(swim_ctx->prot_susp_timeout == 0)
{
#if 0
if(!(g->view.member_states[member_id].is_member))
return;
swim_kill_member(g, member_id, inc_nr);
#endif
return;
}
/* XXX MUTEX */
/* get current swim state for member */
swim_ctx->swim_callbacks.get_member_state(
swim_ctx, member_id, &cur_swim_state);
/* ignore updates for dead members */
if(cur_swim_state.status == SWIM_MEMBER_DEAD)
return;
/* determine if this member is already suspected */
LL_FOREACH_SAFE(suspect_list_p, iter, tmp)
{
if(iter->member_id == member_id)
{
if(inc_nr <= swim_ctx->member_inc_nrs[member_id])
if(inc_nr <= cur_swim_state.inc_nr)
{
/* ignore a suspicion in an incarnation number less than
* or equal to the current suspicion's incarnation
......@@ -388,18 +404,10 @@ static void swim_suspect_member(
}
/* ignore suspicions for a member that is alive in a newer incarnation */
if((suspect_link == NULL) && (inc_nr < swim_ctx->member_inc_nrs[member_id]))
return;
/* if there is no suspicion timeout, just kill the member */
if(swim_ctx->prot_susp_timeout == 0)
{
swim_kill_member(g, member_id, inc_nr);
if((suspect_link == NULL) && (inc_nr < cur_swim_state.inc_nr))
return;
}
SSG_DEBUG(g, "SWIM: member %d SUSPECT (inc_nr=%d)\n",
(int)member_id, (int)inc_nr);
SWIM_DEBUG(swim_ctx, "member %lu SUSPECT (inc_nr=%lu)\n", member_id, inc_nr);
if(suspect_link == NULL)
{
......@@ -417,9 +425,10 @@ static void swim_suspect_member(
/* add to end of suspect list */
LL_APPEND(suspect_list_p, suspect_link);
/* update swim membership state */
/* XXX XXX XXX XXX update swim membership state */
swim_ctx->member_inc_nrs[member_id] = inc_nr;
#if 0
/* add this update to recent update list so it will be piggybacked
* on future protocol messages
*/
......@@ -427,6 +436,7 @@ static void swim_suspect_member(
update.status = SWIM_MEMBER_SUSPECT;
update.inc_nr = inc_nr;
swim_add_recent_member_update(g, update);
#endif
return;
}
......
......@@ -32,13 +32,6 @@ typedef struct swim_member_state
swim_member_status_t status;
} swim_member_state_t;
typedef struct swim_dping_target_info
{
swim_member_id_t id;
hg_addr_t addr;
swim_member_state_t swim_state;
} swim_dping_target_info_t;
#define SWIM_MEMBER_STATE_INIT(__ms) do { \
__ms.inc_nr = 0; \
__ms.status = SWIM_MEMBER_ALIVE; \
......@@ -47,11 +40,28 @@ typedef struct swim_dping_target_info
/* XXX rename once more clear what all is here */
typedef struct swim_group_mgmt_callbacks
{
/* XXX RET VALS */
int (*get_dping_target)(
void *group_data,
swim_dping_target_info_t *target_info
swim_member_id_t *target_id,
swim_member_inc_nr_t *inc_nr,
hg_addr_t *target_addr
);
int (*get_iping_targets)(
void *group_data,
swim_member_id_t *target_ids,
hg_addr_t *target_addrs
);
void (*get_member_addr)(
void *group_data,
swim_member_id_t id,
hg_addr_t *addr
);
void (*get_member_state)(
void *group_data,
swim_member_id_t id,
swim_member_state_t **state
);
/* get_rand_iping_subgroup */
} swim_group_mgmt_callbacks_t;
/* Initialize SWIM */
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment