Commit 3c2ab846 authored by Shane Snyder's avatar Shane Snyder

dynamic joins are now working and tested

parent 5e2e586e
......@@ -36,11 +36,12 @@ typedef uint64_t ssg_member_id_t;
#define SSG_MEMBER_ID_INVALID 0
/* SSG group member update types */
enum ssg_membership_update_type
typedef enum ssg_update_type
{
SSG_MEMBER_ADD = 0,
SSG_MEMBER_REMOVE
};
SSG_MEMBER_JOINED = 0,
SSG_MEMBER_LEFT,
SSG_MEMBER_DIED
} ssg_update_type_t;
typedef struct ssg_member_update
{
......
......@@ -25,15 +25,37 @@ extern "C" {
#define SSG_MAGIC_NR 17321588
#define SSG_GET_SELF_ADDR_STR(__mid, __addr_str) do { \
hg_addr_t __self_addr; \
hg_size_t __size; \
__addr_str = NULL; \
if (margo_addr_self(__mid, &__self_addr) != HG_SUCCESS) break; \
if (margo_addr_to_string(__mid, NULL, &__size, __self_addr) != HG_SUCCESS) { \
margo_addr_free(__mid, __self_addr); \
break; \
} \
if ((__addr_str = malloc(__size)) == NULL) { \
margo_addr_free(__mid, __self_addr); \
break; \
} \
if (margo_addr_to_string(__mid, __addr_str, &__size, __self_addr) != HG_SUCCESS) { \
free(__addr_str); \
__addr_str = NULL; \
margo_addr_free(__mid, __self_addr); \
break; \
} \
margo_addr_free(__mid, __self_addr); \
} while(0)
/* debug printing macro for SSG */
/* TODO: direct debug output to file? */
/* TODO: how do we debug attachers? */
#ifdef DEBUG
#define SSG_DEBUG(__g, __fmt, ...) do { \
double __now = ABT_get_wtime(); \
fprintf(stdout, "[%.6lf] %20"PRIu64" (%s): SSG " __fmt, __now, \
fprintf(g->dbg_log, "[%.6lf] %20"PRIu64" (%s): SSG " __fmt, __now, \
__g->self_id, __g->name, ## __VA_ARGS__); \
fflush(stdout); \
fflush(g->dbg_log); \
} while(0)
#else
#define SSG_DEBUG(__g, __fmt, ...) do { \
......@@ -86,6 +108,9 @@ typedef struct ssg_group
ABT_rwlock lock;
ssg_membership_update_cb update_cb;
void *update_cb_dat;
#ifdef DEBUG
FILE *dbg_log;
#endif
UT_hash_handle hh;
} ssg_group_t;
......@@ -137,6 +162,10 @@ int ssg_group_attach_send(
char ** group_name,
int * group_size,
void ** view_buf);
void ssg_apply_swim_user_updates(
void *group_data,
swim_user_update_t *updates,
hg_size_t update_count);
/* XXX: is this right? can this be a global? */
extern ssg_instance_t *ssg_inst;
......
......@@ -18,6 +18,17 @@
#define SSG_VIEW_BUF_DEF_SIZE (128 * 1024)
#define SSG_USER_UPDATE_SERIALIZE(__type, __data, __size, __update) do { \
__update.size = sizeof(uint8_t) + __size; \
__update.data = malloc(__update.size); \
if (__update.data) { \
void *__p = __update.data; \
*(uint8_t *)__p = __type; \
__p += sizeof(uint8_t); \
memcpy(__p, __data, __size); \
} \
} while(0)
/* SSG RPC types and (de)serialization routines */
/* TODO join and attach are nearly identical -- refactor */
......@@ -59,8 +70,8 @@ DECLARE_MARGO_RPC_HANDLER(ssg_group_leave_recv_ult)
DECLARE_MARGO_RPC_HANDLER(ssg_group_attach_recv_ult)
/* internal helper routine prototypes */
static int ssg_group_view_serialize(
ssg_group_view_t *view, void **buf, hg_size_t *buf_size);
static int ssg_group_serialize(
ssg_group_t *g, void **buf, hg_size_t *buf_size);
/* SSG RPC IDs */
static hg_id_t ssg_group_join_rpc_id;
......@@ -215,56 +226,56 @@ static void ssg_group_join_recv_ult(
void *view_buf = NULL;
hg_size_t view_buf_size;
hg_bulk_t bulk_handle = HG_BULK_NULL;
char *join_addr_str = NULL;
hg_size_t join_addr_str_size = 0;
swim_user_update_t join_update;
int sret;
hg_return_t hret;
if (!ssg_inst) goto fini;
if (!ssg_inst) return;
hgi = margo_get_info(handle);
if (!hgi) goto fini;
if (!hgi) return;
hret = margo_get_input(handle, &join_req);
if (hret != HG_SUCCESS) goto fini;
if (hret != HG_SUCCESS) return;
view_size_requested = margo_bulk_get_size(join_req.bulk_handle);
/* look for the given group in my local table of groups */
HASH_FIND(hh, ssg_inst->group_table, &join_req.group_descriptor.name_hash,
sizeof(uint64_t), g);
if (!g)
{
margo_free_input(handle, &join_req);
goto fini;
}
if (!g) goto fini;
sret = ssg_group_view_serialize(&g->view, &view_buf, &view_buf_size);
if (sret != SSG_SUCCESS)
{
margo_free_input(handle, &join_req);
goto fini;
}
sret = ssg_group_serialize(g, &view_buf, &view_buf_size);
if (sret != SSG_SUCCESS) goto fini;
if (view_size_requested >= view_buf_size)
{
/* if attacher's buf is large enough, transfer the view */
hret = margo_bulk_create(ssg_inst->mid, 1, &view_buf, &view_buf_size,
HG_BULK_READ_ONLY, &bulk_handle);
if (hret != HG_SUCCESS)
{
margo_free_input(handle, &join_req);
goto fini;
}
if (hret != HG_SUCCESS) goto fini;
hret = margo_bulk_transfer(ssg_inst->mid, HG_BULK_PUSH, hgi->addr,
join_req.bulk_handle, 0, bulk_handle, 0, view_buf_size);
if (hret != HG_SUCCESS)
{
margo_free_input(handle, &join_req);
goto fini;
}
}
if (hret != HG_SUCCESS) goto fini;
/* get joining member's address string */
hret = margo_addr_to_string(ssg_inst->mid, NULL, &join_addr_str_size, hgi->addr);
if (hret != HG_SUCCESS) goto fini;
join_addr_str = malloc(join_addr_str_size);
if (join_addr_str == NULL) goto fini;
hret = margo_addr_to_string(ssg_inst->mid, join_addr_str, &join_addr_str_size, hgi->addr);
if (hret != HG_SUCCESS) goto fini;
/* create an SSG join update and register with SWIM to be gossiped */
SSG_USER_UPDATE_SERIALIZE(SSG_MEMBER_JOINED, join_addr_str,
join_addr_str_size, join_update);
swim_register_user_update(g->swim_ctx, join_update);
/* XXX what else? need to add to view/target list */
printf("***SDS: received JOINNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN REQUESTTTTTTTTTTTTTT\n");
/* apply group join locally */
ssg_apply_swim_user_updates(g, &join_update, 1);
}
/* set the response and send back */
join_resp.group_name = g->name;
......@@ -272,9 +283,10 @@ static void ssg_group_join_recv_ult(
join_resp.view_buf_size = view_buf_size;
margo_respond(handle, &join_resp);
margo_free_input(handle, &join_req);
fini:
free(view_buf);
free(join_addr_str);
margo_free_input(handle, &join_req);
if (handle != HG_HANDLE_NULL) margo_destroy(handle);
if (bulk_handle != HG_BULK_NULL) margo_bulk_free(bulk_handle);
......@@ -445,7 +457,7 @@ static void ssg_group_attach_recv_ult(
goto fini;
}
sret = ssg_group_view_serialize(&g->view, &view_buf, &view_buf_size);
sret = ssg_group_serialize(g, &view_buf, &view_buf_size);
if (sret != SSG_SUCCESS)
{
margo_free_input(handle, &attach_req);
......@@ -488,37 +500,48 @@ fini:
}
DEFINE_MARGO_RPC_HANDLER(ssg_group_attach_recv_ult)
static int ssg_group_view_serialize(
ssg_group_view_t *view, void **buf, hg_size_t *buf_size)
static int ssg_group_serialize(
ssg_group_t *g, void **buf, hg_size_t *buf_size)
{
char *self_addr_str;
ssg_member_state_t *member_state, *tmp;
hg_size_t view_buf_size = 0;
void *view_buf;
hg_size_t group_buf_size = 0;
void *group_buf;
void *buf_p, *str_p;
*buf = NULL;
*buf_size = 0;
/* first determine view size */
HASH_ITER(hh, view->member_map, member_state, tmp)
SSG_GET_SELF_ADDR_STR(ssg_inst->mid, self_addr_str);
if (!self_addr_str) return SSG_FAILURE;
/* first determine size */
group_buf_size = strlen(self_addr_str) + 1;
HASH_ITER(hh, g->view.member_map, member_state, tmp)
{
view_buf_size += strlen(member_state->addr_str) + 1;
group_buf_size += strlen(member_state->addr_str) + 1;
}
view_buf = malloc(view_buf_size);
if(!view_buf)
group_buf = malloc(group_buf_size);
if(!group_buf)
{
free(self_addr_str);
return SSG_FAILURE;
}
buf_p = view_buf;
HASH_ITER(hh, view->member_map, member_state, tmp)
buf_p = group_buf;
strcpy(buf_p, self_addr_str);
buf_p += strlen(self_addr_str) + 1;
HASH_ITER(hh, g->view.member_map, member_state, tmp)
{
str_p = member_state->addr_str;
strcpy(buf_p, str_p);
buf_p += strlen(member_state->addr_str) + 1;
}
*buf = view_buf;
*buf_size = view_buf_size;
*buf = group_buf;
*buf_size = group_buf_size;
free(self_addr_str);
return SSG_SUCCESS;
}
......
This diff is collapsed.
......@@ -60,9 +60,12 @@ struct swim_context
hg_addr_t iping_target_addrs[SWIM_MAX_SUBGROUP_SIZE];
int iping_target_ndx;
int ping_target_acked;
void *suspect_list;
void *recent_update_list;
int shutdown_flag;
/* list of currently supspected SWIM members */
void *suspect_list;
/* lists of SWIM membership updates and user-supplied updates */
void *swim_update_list;
void *user_update_list;
/* argobots pool for launching SWIM threads */
ABT_pool swim_pool;
/* mutex for modifying SWIM group state */
......@@ -79,12 +82,16 @@ void swim_dping_send_ult(
void swim_iping_send_ult(
void * t_arg);
/* SWIM membership update function prototypes */
void swim_retrieve_membership_updates(
/* SWIM update function prototypes */
void swim_retrieve_member_updates(
swim_context_t *swim_ctx,
swim_member_update_t *updates,
hg_size_t *update_count);
void swim_apply_membership_updates(
void swim_retrieve_user_updates(
swim_context_t *swim_ctx,
swim_user_update_t *updates,
hg_size_t *update_count);
void swim_apply_member_updates(
swim_context_t *swim_ctx,
swim_member_update_t *updates,
hg_size_t update_count);
......
......@@ -32,11 +32,15 @@ typedef struct swim_message_s
{
swim_member_id_t source_id;
swim_member_inc_nr_t source_inc_nr;
hg_size_t pb_buf_count;
swim_member_update_t pb_buf[SWIM_MAX_PIGGYBACK_ENTRIES]; //TODO: dynamic array?
hg_size_t swim_pb_buf_count;
hg_size_t user_pb_buf_count;
swim_member_update_t swim_pb_buf[SWIM_MAX_PIGGYBACK_ENTRIES]; //TODO: dynamic array?
swim_user_update_t user_pb_buf[SWIM_MAX_PIGGYBACK_ENTRIES]; //TODO: dynamic array?
} swim_message_t;
/* HG encode/decode routines for SWIM RPCs */
static hg_return_t hg_proc_swim_user_update_t(
hg_proc_t proc, void *data);
static hg_return_t hg_proc_swim_message_t(
hg_proc_t proc, void *data);
......@@ -344,9 +348,11 @@ static void swim_pack_message(swim_context_t *swim_ctx, swim_message_t *msg)
msg->source_id = swim_ctx->self_id;
msg->source_inc_nr = swim_ctx->self_inc_nr;
/* piggyback a set of membership states on this message */
msg->pb_buf_count = SWIM_MAX_PIGGYBACK_ENTRIES;
swim_retrieve_membership_updates(swim_ctx, msg->pb_buf, &msg->pb_buf_count);
/* piggyback SWIM updates on the message */
msg->swim_pb_buf_count = SWIM_MAX_PIGGYBACK_ENTRIES;
msg->user_pb_buf_count = SWIM_MAX_PIGGYBACK_ENTRIES;
swim_retrieve_member_updates(swim_ctx, msg->swim_pb_buf, &msg->swim_pb_buf_count);
swim_retrieve_user_updates(swim_ctx, msg->user_pb_buf, &msg->user_pb_buf_count);
return;
}
......@@ -359,10 +365,16 @@ static void swim_unpack_message(swim_context_t *swim_ctx, swim_message_t *msg)
sender_update.id = msg->source_id;
sender_update.state.status = SWIM_MEMBER_ALIVE;
sender_update.state.inc_nr = msg->source_inc_nr;
swim_apply_membership_updates(swim_ctx, &sender_update, 1);
swim_apply_member_updates(swim_ctx, &sender_update, 1);
/* apply SWIM updates */
if(msg->swim_pb_buf_count > 0)
swim_apply_member_updates(swim_ctx, msg->swim_pb_buf, msg->swim_pb_buf_count);
/* update membership status using piggybacked membership updates */
swim_apply_membership_updates(swim_ctx, msg->pb_buf, msg->pb_buf_count);
/* apply user updates */
if(msg->user_pb_buf_count > 0)
swim_ctx->swim_callbacks.apply_user_updates(swim_ctx->group_data,
msg->user_pb_buf, msg->user_pb_buf_count);
return;
}
......@@ -389,15 +401,30 @@ static hg_return_t hg_proc_swim_message_t(hg_proc_t proc, void *data)
hret = HG_PROTOCOL_ERROR;
return hret;
}
hret = hg_proc_hg_size_t(proc, &(msg->pb_buf_count));
hret = hg_proc_hg_size_t(proc, &(msg->swim_pb_buf_count));
if(hret != HG_SUCCESS)
{
hret = HG_PROTOCOL_ERROR;
return hret;
}
hret = hg_proc_hg_size_t(proc, &(msg->user_pb_buf_count));
if(hret != HG_SUCCESS)
{
hret = HG_PROTOCOL_ERROR;
return hret;
}
for(i = 0; i < msg->pb_buf_count; i++)
for(i = 0; i < msg->swim_pb_buf_count; i++)
{
hret = hg_proc_swim_member_update_t(proc, &(msg->pb_buf[i]));
hret = hg_proc_swim_member_update_t(proc, &(msg->swim_pb_buf[i]));
if(hret != HG_SUCCESS)
{
hret = HG_PROTOCOL_ERROR;
return hret;
}
}
for(i = 0; i < msg->user_pb_buf_count; i++)
{
hret = hg_proc_swim_user_update_t(proc, &(msg->user_pb_buf[i]));
if(hret != HG_SUCCESS)
{
hret = HG_PROTOCOL_ERROR;
......@@ -418,24 +445,99 @@ static hg_return_t hg_proc_swim_message_t(hg_proc_t proc, void *data)
hret = HG_PROTOCOL_ERROR;
return hret;
}
hret = hg_proc_hg_size_t(proc, &(msg->pb_buf_count));
hret = hg_proc_hg_size_t(proc, &(msg->swim_pb_buf_count));
if(hret != HG_SUCCESS)
{
hret = HG_PROTOCOL_ERROR;
return hret;
}
hret = hg_proc_hg_size_t(proc, &(msg->user_pb_buf_count));
if(hret != HG_SUCCESS)
{
hret = HG_PROTOCOL_ERROR;
return hret;
}
for(i = 0; i < msg->pb_buf_count; i++)
for(i = 0; i < msg->swim_pb_buf_count; i++)
{
hret = hg_proc_swim_member_update_t(proc, &(msg->pb_buf[i]));
hret = hg_proc_swim_member_update_t(proc, &(msg->swim_pb_buf[i]));
if(hret != HG_SUCCESS)
{
hret = HG_PROTOCOL_ERROR;
return hret;
}
}
for(i = 0; i < msg->user_pb_buf_count; i++)
{
hret = hg_proc_swim_user_update_t(proc, &(msg->user_pb_buf[i]));
if(hret != HG_SUCCESS)
{
hret = HG_PROTOCOL_ERROR;
return hret;
}
}
break;
case HG_FREE:
for(i = 0; i < msg->user_pb_buf_count; i++)
{
hret = hg_proc_swim_user_update_t(proc, &(msg->user_pb_buf[i]));
if(hret != HG_SUCCESS)
{
hret = HG_PROTOCOL_ERROR;
return hret;
}
}
hret = HG_SUCCESS;
break;
default:
break;
}
return(hret);
}
static hg_return_t hg_proc_swim_user_update_t(hg_proc_t proc, void *data)
{
swim_user_update_t *update = (swim_user_update_t *)data;
hg_return_t hret = HG_PROTOCOL_ERROR;
switch(hg_proc_get_op(proc))
{
case HG_ENCODE:
hret = hg_proc_hg_size_t(proc, &(update->size));
if(hret != HG_SUCCESS)
{
hret = HG_PROTOCOL_ERROR;
return hret;
}
hret = hg_proc_memcpy(proc, update->data, update->size);
if(hret != HG_SUCCESS)
{
hret = HG_PROTOCOL_ERROR;
return hret;
}
break;
case HG_DECODE:
hret = hg_proc_hg_size_t(proc, &(update->size));
if(hret != HG_SUCCESS)
{
hret = HG_PROTOCOL_ERROR;
return hret;
}
update->data = malloc(update->size);
if(!update->data)
{
hret = HG_NOMEM_ERROR;
return hret;
}
hret = hg_proc_memcpy(proc, update->data, update->size);
if(hret != HG_SUCCESS)
{
hret = HG_PROTOCOL_ERROR;
return hret;
}
break;
case HG_FREE:
/* do nothing */
free(update->data);
hret = HG_SUCCESS;
break;
default:
......
This diff is collapsed.
......@@ -40,6 +40,13 @@ typedef struct swim_member_update
swim_member_state_t state;
} swim_member_update_t;
/* generic SWIM user update */
typedef struct swim_user_update
{
hg_size_t size;
void *data;
} swim_user_update_t;
#define SWIM_MEMBER_STATE_INIT(__ms) do { \
__ms.inc_nr = 0; \
__ms.status = SWIM_MEMBER_ALIVE; \
......@@ -120,6 +127,11 @@ typedef struct swim_group_mgmt_callbacks
void *group_data,
swim_member_update_t update
);
void (*apply_user_updates)(
void *group_data,
swim_user_update_t *updates,
hg_size_t update_count
);
} swim_group_mgmt_callbacks_t;
/**
......@@ -134,7 +146,7 @@ typedef struct swim_group_mgmt_callbacks
*/
swim_context_t * swim_init(
margo_instance_id mid,
void * group_data,
void *group_data,
swim_member_id_t self_id,
swim_group_mgmt_callbacks_t swim_callbacks,
int active);
......@@ -145,7 +157,14 @@ swim_context_t * swim_init(
* @param[in] swim_ctx SWIM context pointer
*/
void swim_finalize(
swim_context_t * swim_ctx);
swim_context_t *swim_ctx);
/**
*
*/
void swim_register_user_update(
swim_context_t *swim_ctx,
swim_user_update_t update);
#ifdef __cplusplus
}
......
......@@ -8,11 +8,13 @@ source $srcdir/tests/test-util.sh
TMPOUT=$($MKTEMP -d --tmpdir test-XXXXXX)
#export SSG_DEBUG_LOGDIR=$TMPOUT
# launch initial group, storing GID
export SSG_GROUP_LAUNCH_NAME=simplest-group
export SSG_GROUP_LAUNCH_DURATION=10
export SSG_GROUP_LAUNCH_GIDFILE=$TMPOUT/gid.out
launch_ssg_group_mpi 4 na+sm
export SSG_GROUP_LAUNCH_GIDFILE=gid.out
launch_ssg_group_mpi 4 na+sm &
if [ $? -ne 0 ]; then
wait
rm -rf $TMPOUT
......@@ -23,7 +25,7 @@ sleep 2
# try to join running group
export SSG_GROUP_LAUNCH_DURATION=8
join_ssg_group na+sm $SSG_GROUP_LAUNCH_GIDFILE
join_ssg_group na+sm $SSG_GROUP_LAUNCH_GIDFILE &
if [ $? -ne 0 ]; then
wait
rm -rf $TMPOUT
......@@ -36,5 +38,5 @@ if [ $? -ne 0 ]; then
exit 1
fi
rm -rf $TMPOUT
#rm -rf $TMPOUT
exit 0
......@@ -9,7 +9,7 @@ source $srcdir/tests/test-util.sh
# launch a group and wait for termination
export SSG_GROUP_LAUNCH_NAME=simplest-group
export SSG_GROUP_LAUNCH_DURATION=10
launch_ssg_group_mpi 4 na+sm
launch_ssg_group_mpi 4 na+sm &
if [ $? -ne 0 ]; then
wait
exit 1
......
......@@ -101,6 +101,9 @@ int main(int argc, char *argv[])
sret = ssg_init(mid);
DIE_IF(sret != SSG_SUCCESS, "ssg_init");
/* load GID from file */
ssg_group_id_load(opts.gid_file, &in_g_id);
/* XXX do we want to use callback for testing anything about group??? */
out_g_id = ssg_group_join(in_g_id, NULL, NULL);
DIE_IF(out_g_id == SSG_GROUP_ID_NULL, "ssg_group_join");
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment