Commit 16208a65 authored by Shane Snyder's avatar Shane Snyder

finish implementing ssg dping/iping paths

parent c6dd1a84
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
#include "swim-fd/swim-fd.h" #include "swim-fd/swim-fd.h"
#include "uthash.h" #include "uthash.h"
#include "utlist.h" #include "utlist.h"
#include "utarray.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
...@@ -48,7 +49,6 @@ typedef struct ssg_member_state ...@@ -48,7 +49,6 @@ typedef struct ssg_member_state
char *addr_str; char *addr_str;
hg_addr_t addr; hg_addr_t addr;
swim_member_state_t swim_state; swim_member_state_t swim_state;
struct ssg_member_state *next;
UT_hash_handle hh; UT_hash_handle hh;
} ssg_member_state_t; } ssg_member_state_t;
...@@ -71,11 +71,13 @@ typedef struct ssg_group_view ...@@ -71,11 +71,13 @@ typedef struct ssg_group_view
typedef struct ssg_group typedef struct ssg_group
{ {
char *name; char *name;
ssg_member_id_t self_id;
ssg_group_view_t view; ssg_group_view_t view;
ssg_member_id_t self_id;
UT_array *nondead_member_list;
unsigned int nondead_member_count;
unsigned int dping_target_ndx;
ssg_group_descriptor_t *descriptor; ssg_group_descriptor_t *descriptor;
swim_context_t *swim_ctx; swim_context_t *swim_ctx;
ssg_member_state_t *member_list;
ssg_membership_update_cb update_cb; ssg_membership_update_cb update_cb;
void *update_cb_dat; void *update_cb_dat;
UT_hash_handle hh; UT_hash_handle hh;
......
...@@ -130,6 +130,7 @@ static int ssg_get_swim_dping_target( ...@@ -130,6 +130,7 @@ static int ssg_get_swim_dping_target(
hg_addr_t *target_addr); hg_addr_t *target_addr);
static int ssg_get_swim_iping_targets( static int ssg_get_swim_iping_targets(
void *group_data, void *group_data,
swim_member_id_t dping_target_id,
int *num_targets, int *num_targets,
swim_member_id_t *target_ids, swim_member_id_t *target_ids,
hg_addr_t *target_addrs); hg_addr_t *target_addrs);
...@@ -145,7 +146,7 @@ static void ssg_apply_swim_member_update( ...@@ -145,7 +146,7 @@ static void ssg_apply_swim_member_update(
void *group_data, void *group_data,
swim_member_update_t update); swim_member_update_t update);
static void ssg_gen_rand_member_list( static void ssg_shuffle_member_list(
ssg_group_t *g); ssg_group_t *g);
static int ssg_get_swim_dping_target( static int ssg_get_swim_dping_target(
...@@ -155,53 +156,79 @@ static int ssg_get_swim_dping_target( ...@@ -155,53 +156,79 @@ static int ssg_get_swim_dping_target(
hg_addr_t *target_addr) hg_addr_t *target_addr)
{ {
ssg_group_t *g = (ssg_group_t *)group_data; ssg_group_t *g = (ssg_group_t *)group_data;
ssg_member_state_t *target_ms; ssg_member_state_t **target_ms_p;
assert(g != NULL); assert(g != NULL);
/* if current member list is empty, generate a new random one */ /* XXX MUTEX */
if (g->member_list == NULL)
if (g->dping_target_ndx == 0)
{ {
ssg_gen_rand_member_list(g); /* reshuffle member list */
if(g->member_list == NULL) ssg_shuffle_member_list(g);
return -1;
} }
/* pull random member off head of list and return addr */ target_ms_p = (ssg_member_state_t **)utarray_eltptr(
target_ms = g->member_list; g->nondead_member_list, g->dping_target_ndx);
LL_DELETE(g->member_list, target_ms); *target_id = (swim_member_id_t)(*target_ms_p)->id;
*target_id = (swim_member_id_t)target_ms->id; *target_inc_nr = (*target_ms_p)->swim_state.inc_nr;
*target_inc_nr = target_ms->swim_state.inc_nr; *target_addr = (*target_ms_p)->addr;
*target_addr = target_ms->addr;
return 0;
}
static void ssg_gen_rand_member_list(ssg_group_t *g)
{
ssg_member_state_t *ms, *tmp;
/* XXX generate random LL permutation from membership map */ /* point at next dping target in the view */
HASH_ITER(hh, g->view.member_map, ms, tmp) g->dping_target_ndx++;
{ if (g->dping_target_ndx == g->nondead_member_count)
LL_APPEND(g->member_list, ms); g->dping_target_ndx = 0;
}
return; return 0;
} }
static int ssg_get_swim_iping_targets( static int ssg_get_swim_iping_targets(
void *group_data, void *group_data,
swim_member_id_t dping_target_id,
int *num_targets, int *num_targets,
swim_member_id_t *target_id, swim_member_id_t *target_ids,
hg_addr_t *target_addrs) hg_addr_t *target_addrs)
{ {
ssg_group_t *g = (ssg_group_t *)group_data; ssg_group_t *g = (ssg_group_t *)group_data;
int max_targets = *num_targets; int max_targets = *num_targets;
int iping_target_count = 0;
int i = 0;
unsigned int r_start, r_ndx;
ssg_member_state_t **r_ms_p;
assert(g != NULL); assert(g != NULL);
*num_targets = 0; *num_targets = 0;
/* XXX MUTEX */
/* pick random index in the nondead list, and pull out a set of iping
* targets starting from that index
*/
r_start = rand() % g->nondead_member_count;
while (iping_target_count < max_targets)
{
r_ndx = (r_start + i) % g->nondead_member_count;
/* if we've iterated through the entire nondead list, stop */
if ((i > 0 ) && (r_ndx == r_start)) break;
r_ms_p = (ssg_member_state_t **)utarray_eltptr(g->nondead_member_list, r_ndx);
/* do not select the dping target as an iping target */
if ((swim_member_id_t)(*r_ms_p)->id == dping_target_id)
{
i++;
continue;
}
target_ids[iping_target_count] = (swim_member_id_t)(*r_ms_p)->id;
target_addrs[iping_target_count] = (*r_ms_p)->addr;
iping_target_count++;
i++;
}
*num_targets = iping_target_count;
return 0; return 0;
} }
...@@ -260,9 +287,7 @@ static void ssg_apply_swim_member_update( ...@@ -260,9 +287,7 @@ static void ssg_apply_swim_member_update(
if (ms) if (ms)
{ {
/* update group, but don't completely remove state */ /* update group, but don't completely remove state */
LL_DELETE(g->member_list, ms);
margo_addr_free(ssg_inst->mid, ms->addr); margo_addr_free(ssg_inst->mid, ms->addr);
g->view.size--;
ssg_update.id = ssg_id; ssg_update.id = ssg_id;
ssg_update.type = SSG_MEMBER_REMOVE; ssg_update.type = SSG_MEMBER_REMOVE;
} }
...@@ -279,6 +304,28 @@ static void ssg_apply_swim_member_update( ...@@ -279,6 +304,28 @@ static void ssg_apply_swim_member_update(
return; return;
} }
static void ssg_shuffle_member_list(
ssg_group_t *g)
{
unsigned int i, r;
ssg_member_state_t **tmp_ms;
UT_array *list = g->nondead_member_list;
unsigned int len = g->nondead_member_count;
/* run fisher-yates shuffle over list of nondead members */
for (i = len - 1; i > 0; i--)
{
r = rand() % (i + 1);
tmp_ms = (ssg_member_state_t **)utarray_eltptr(list, r);
utarray_erase(list, r, 1);
utarray_insert(list, *(ssg_member_state_t **)utarray_eltptr(list, i), r);
utarray_erase(list, i, 1);
utarray_insert(list, *tmp_ms, i);
}
return;
}
ssg_group_id_t ssg_group_create( ssg_group_id_t ssg_group_create(
const char * group_name, const char * group_name,
const char * const group_addr_strs[], const char * const group_addr_strs[],
...@@ -290,6 +337,7 @@ ssg_group_id_t ssg_group_create( ...@@ -290,6 +337,7 @@ ssg_group_id_t ssg_group_create(
hg_addr_t self_addr = HG_ADDR_NULL; hg_addr_t self_addr = HG_ADDR_NULL;
ssg_group_descriptor_t *tmp_descriptor; ssg_group_descriptor_t *tmp_descriptor;
ssg_group_t *g = NULL; ssg_group_t *g = NULL;
ssg_member_state_t *ms, *tmp_ms;
hg_return_t hret; hg_return_t hret;
int sret; int sret;
ssg_group_id_t group_id = SSG_GROUP_ID_NULL; ssg_group_id_t group_id = SSG_GROUP_ID_NULL;
...@@ -338,8 +386,15 @@ ssg_group_id_t ssg_group_create( ...@@ -338,8 +386,15 @@ ssg_group_id_t ssg_group_create(
goto fini; goto fini;
} }
/* generate random list permutation of the member map */ /* create a list of all nondead member states (used for shuffling) */
ssg_gen_rand_member_list(g); UT_icd ms_icd = {sizeof(ssg_member_state_t *), NULL, NULL, NULL};
utarray_new(g->nondead_member_list, &ms_icd);
utarray_reserve(g->nondead_member_list, g->view.size);
HASH_ITER(hh, g->view.member_map, ms, tmp_ms)
{
utarray_push_back(g->nondead_member_list, ms);
}
g->nondead_member_count = g->view.size;
/* initialize swim failure detector */ /* initialize swim failure detector */
// TODO: we should probably barrier or sync somehow to avoid rpc failures // TODO: we should probably barrier or sync somehow to avoid rpc failures
...@@ -1301,6 +1356,7 @@ static void ssg_group_lookup_ult( ...@@ -1301,6 +1356,7 @@ static void ssg_group_lookup_ult(
&l->member_state->addr); &l->member_state->addr);
if (l->out == HG_SUCCESS) if (l->out == HG_SUCCESS)
{ {
/* XXX MUTEX */
HASH_ADD(hh, l->view->member_map, id, sizeof(ssg_member_id_t), HASH_ADD(hh, l->view->member_map, id, sizeof(ssg_member_id_t),
l->member_state); l->member_state);
} }
......
...@@ -24,7 +24,7 @@ extern "C" { ...@@ -24,7 +24,7 @@ extern "C" {
#define SWIM_MAX_PIGGYBACK_ENTRIES 8 #define SWIM_MAX_PIGGYBACK_ENTRIES 8
#define SWIM_MAX_PIGGYBACK_TX_COUNT 50 #define SWIM_MAX_PIGGYBACK_TX_COUNT 50
/* debug printing macro for SSG */ /* debug printing macro for SWIM */
#ifdef DEBUG #ifdef DEBUG
#define SWIM_DEBUG(__swim_ctx, __fmt, ...) do { \ #define SWIM_DEBUG(__swim_ctx, __fmt, ...) do { \
double __now = ABT_get_wtime(); \ double __now = ABT_get_wtime(); \
......
...@@ -194,8 +194,8 @@ static void swim_tick_ult( ...@@ -194,8 +194,8 @@ static void swim_tick_ult(
/* get a random subgroup of members to send indirect pings to */ /* get a random subgroup of members to send indirect pings to */
int iping_target_count = swim_ctx->prot_subgroup_sz; int iping_target_count = swim_ctx->prot_subgroup_sz;
swim_ctx->swim_callbacks.get_iping_targets( swim_ctx->swim_callbacks.get_iping_targets(
swim_ctx->group_data, &iping_target_count, swim_ctx->iping_target_ids, swim_ctx->group_data, swim_ctx->dping_target_id, &iping_target_count,
swim_ctx->iping_target_addrs); swim_ctx->iping_target_ids, swim_ctx->iping_target_addrs);
if(iping_target_count == 0) if(iping_target_count == 0)
{ {
/* no available subgroup members, back out */ /* no available subgroup members, back out */
......
...@@ -71,6 +71,7 @@ typedef struct swim_group_mgmt_callbacks ...@@ -71,6 +71,7 @@ typedef struct swim_group_mgmt_callbacks
* management layer to send indirect ping requests to. * management layer to send indirect ping requests to.
* *
* @param[in] group_data void pointer to group managment data * @param[in] group_data void pointer to group managment data
* @param[in] dping_target_id corresponding dping target ID
* @param[in/out] num_targets on input, maximum number of indirect ping * @param[in/out] num_targets on input, maximum number of indirect ping
* targets to select. on output, the actual * targets to select. on output, the actual
* number of selected targets * number of selected targets
...@@ -80,6 +81,7 @@ typedef struct swim_group_mgmt_callbacks ...@@ -80,6 +81,7 @@ typedef struct swim_group_mgmt_callbacks
*/ */
int (*get_iping_targets)( int (*get_iping_targets)(
void *group_data, void *group_data,
swim_member_id_t dping_target_id,
int *num_targets, int *num_targets,
swim_member_id_t *target_ids, swim_member_id_t *target_ids,
hg_addr_t *target_addrs hg_addr_t *target_addrs
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment