Commit 521d75fa authored by Philip Carns's avatar Philip Carns
Browse files

create initialization instance and move globals

parent 4f88a849
...@@ -20,18 +20,25 @@ ...@@ -20,18 +20,25 @@
* The HG forward call is executed using asynchronous operations. * The HG forward call is executed using asynchronous operations.
*/ */
struct run_my_rpc_args
{
int val;
margo_instance_id mid;
};
static void run_my_rpc(void *_arg); static void run_my_rpc(void *_arg);
static hg_id_t my_rpc_id; static hg_id_t my_rpc_id;
int main(int argc, char **argv) int main(int argc, char **argv)
{ {
int values[4]; struct run_my_rpc_args args[4];
ABT_thread threads[4]; ABT_thread threads[4];
int i; int i;
int ret; int ret;
ABT_xstream xstream; ABT_xstream xstream;
ABT_pool pool; ABT_pool pool;
margo_instance_id mid;
ret = ABT_init(argc, argv); ret = ABT_init(argc, argv);
if(ret != 0) if(ret != 0)
...@@ -65,18 +72,19 @@ int main(int argc, char **argv) ...@@ -65,18 +72,19 @@ int main(int argc, char **argv)
/* initialize /* initialize
* note: address here is really just being used to identify transport * note: address here is really just being used to identify transport
*/ */
margo_init(NA_FALSE, "tcp://localhost:1234"); mid = margo_init(NA_FALSE, "tcp://localhost:1234");
/* register RPC */ /* register RPC */
my_rpc_id = my_rpc_register(); my_rpc_id = my_rpc_register(mid);
for(i=0; i<4; i++) for(i=0; i<4; i++)
{ {
values[i] = i; args[i].val = i;
/* Each fiber gets a pointer to an element of the values array to use args[i].mid = mid;
/* Each fiber gets a pointer to an element of the array to use
* as input for the run_my_rpc() function. * as input for the run_my_rpc() function.
*/ */
ret = ABT_thread_create(pool, run_my_rpc, &values[i], ret = ABT_thread_create(pool, run_my_rpc, &args[i],
ABT_THREAD_ATTR_NULL, &threads[i]); ABT_THREAD_ATTR_NULL, &threads[i]);
if(ret != 0) if(ret != 0)
{ {
...@@ -105,7 +113,7 @@ int main(int argc, char **argv) ...@@ -105,7 +113,7 @@ int main(int argc, char **argv)
} }
} }
margo_finalize(); margo_finalize(mid);
ABT_finalize(); ABT_finalize();
return(0); return(0);
...@@ -113,7 +121,7 @@ int main(int argc, char **argv) ...@@ -113,7 +121,7 @@ int main(int argc, char **argv)
static void run_my_rpc(void *_arg) static void run_my_rpc(void *_arg)
{ {
int* val = (int*)_arg; struct run_my_rpc_args *arg = _arg;
na_addr_t svr_addr = NA_ADDR_NULL; na_addr_t svr_addr = NA_ADDR_NULL;
hg_handle_t handle; hg_handle_t handle;
my_rpc_in_t in; my_rpc_in_t in;
...@@ -123,7 +131,7 @@ static void run_my_rpc(void *_arg) ...@@ -123,7 +131,7 @@ static void run_my_rpc(void *_arg)
void* buffer; void* buffer;
struct hg_info *hgi; struct hg_info *hgi;
printf("ULT [%d] running.\n", *val); printf("ULT [%d] running.\n", arg->val);
/* allocate buffer for bulk transfer */ /* allocate buffer for bulk transfer */
size = 512; size = 512;
...@@ -132,11 +140,11 @@ static void run_my_rpc(void *_arg) ...@@ -132,11 +140,11 @@ static void run_my_rpc(void *_arg)
sprintf((char*)buffer, "Hello world!\n"); sprintf((char*)buffer, "Hello world!\n");
/* find addr for server */ /* find addr for server */
ret = margo_addr_lookup("tcp://localhost:1234", &svr_addr); ret = margo_addr_lookup(arg->mid, "tcp://localhost:1234", &svr_addr);
assert(ret == 0); assert(ret == 0);
/* create handle */ /* create handle */
ret = margo_create_handle(svr_addr, my_rpc_id, &handle); ret = margo_create_handle(arg->mid, svr_addr, my_rpc_id, &handle);
assert(ret == 0); assert(ret == 0);
/* register buffer for rdma/bulk access by server */ /* register buffer for rdma/bulk access by server */
...@@ -149,8 +157,8 @@ static void run_my_rpc(void *_arg) ...@@ -149,8 +157,8 @@ static void run_my_rpc(void *_arg)
/* Send rpc. Note that we are also transmitting the bulk handle in the /* Send rpc. Note that we are also transmitting the bulk handle in the
* input struct. It was set above. * input struct. It was set above.
*/ */
in.input_val = *((int*)(_arg)); in.input_val = arg->val;
margo_forward(handle, &in); margo_forward(arg->mid, handle, &in);
/* decode response */ /* decode response */
ret = HG_Get_output(handle, &out); ret = HG_Get_output(handle, &out);
...@@ -164,7 +172,7 @@ static void run_my_rpc(void *_arg) ...@@ -164,7 +172,7 @@ static void run_my_rpc(void *_arg)
HG_Destroy(handle); HG_Destroy(handle);
free(buffer); free(buffer);
printf("ULT [%d] done.\n", *val); printf("ULT [%d] done.\n", arg->val);
return; return;
} }
...@@ -32,6 +32,7 @@ static void my_rpc_ult(void *_arg) ...@@ -32,6 +32,7 @@ static void my_rpc_ult(void *_arg)
struct hg_info *hgi; struct hg_info *hgi;
int fd; int fd;
char filename[256]; char filename[256];
margo_instance_id mid;
ret = HG_Get_input(*handle, &in); ret = HG_Get_input(*handle, &in);
assert(ret == HG_SUCCESS); assert(ret == HG_SUCCESS);
...@@ -51,8 +52,10 @@ static void my_rpc_ult(void *_arg) ...@@ -51,8 +52,10 @@ static void my_rpc_ult(void *_arg)
&size, HG_BULK_WRITE_ONLY, &bulk_handle); &size, HG_BULK_WRITE_ONLY, &bulk_handle);
assert(ret == 0); assert(ret == 0);
mid = margo_hg_class_to_instance(hgi->hg_class);
/* do bulk transfer from client to server */ /* do bulk transfer from client to server */
ret = margo_bulk_transfer(hgi->bulk_context, HG_BULK_PULL, ret = margo_bulk_transfer(mid, hgi->bulk_context, HG_BULK_PULL,
hgi->addr, in.bulk_handle, 0, hgi->addr, in.bulk_handle, 0,
bulk_handle, 0, size); bulk_handle, 0, size);
assert(ret == 0); assert(ret == 0);
...@@ -82,12 +85,12 @@ static void my_rpc_ult(void *_arg) ...@@ -82,12 +85,12 @@ static void my_rpc_ult(void *_arg)
} }
DEFINE_ARGO_RPC_HANDLER(my_rpc_ult) DEFINE_ARGO_RPC_HANDLER(my_rpc_ult)
hg_id_t my_rpc_register(void) hg_id_t my_rpc_register(margo_instance_id mid)
{ {
hg_class_t* hg_class; hg_class_t* hg_class;
hg_id_t tmp; hg_id_t tmp;
hg_class = margo_get_class(); hg_class = margo_get_class(mid);
tmp = MERCURY_REGISTER(hg_class, "my_rpc", my_rpc_in_t, my_rpc_out_t, tmp = MERCURY_REGISTER(hg_class, "my_rpc", my_rpc_in_t, my_rpc_out_t,
my_rpc_ult_handler); my_rpc_ult_handler);
......
...@@ -16,6 +16,6 @@ MERCURY_GEN_PROC(my_rpc_in_t, ...@@ -16,6 +16,6 @@ MERCURY_GEN_PROC(my_rpc_in_t,
((int32_t)(input_val))\ ((int32_t)(input_val))\
((hg_bulk_t)(bulk_handle))) ((hg_bulk_t)(bulk_handle)))
hg_id_t my_rpc_register(void); hg_id_t my_rpc_register(margo_instance_id mid);
#endif /* __MY_RPC */ #endif /* __MY_RPC */
...@@ -22,6 +22,7 @@ int main(int argc, char **argv) ...@@ -22,6 +22,7 @@ int main(int argc, char **argv)
int ret; int ret;
ABT_eventual eventual; ABT_eventual eventual;
int *shutdown; int *shutdown;
margo_instance_id mid;
ret = ABT_init(argc, argv); ret = ABT_init(argc, argv);
if(ret != 0) if(ret != 0)
...@@ -38,10 +39,10 @@ int main(int argc, char **argv) ...@@ -38,10 +39,10 @@ int main(int argc, char **argv)
return(-1); return(-1);
} }
margo_init(NA_TRUE, "tcp://localhost:1234"); mid = margo_init(NA_TRUE, "tcp://localhost:1234");
/* register RPC */ /* register RPC */
my_rpc_register(); my_rpc_register(mid);
/* suspend this ULT until someone tells us to shut down */ /* suspend this ULT until someone tells us to shut down */
ret = ABT_eventual_create(sizeof(*shutdown), &eventual); ret = ABT_eventual_create(sizeof(*shutdown), &eventual);
...@@ -53,7 +54,7 @@ int main(int argc, char **argv) ...@@ -53,7 +54,7 @@ int main(int argc, char **argv)
ABT_eventual_wait(eventual, (void**)&shutdown); ABT_eventual_wait(eventual, (void**)&shutdown);
margo_finalize(); margo_finalize(mid);
ABT_finalize(); ABT_finalize();
return(0); return(0);
......
...@@ -13,6 +13,8 @@ ...@@ -13,6 +13,8 @@
#include <abt.h> #include <abt.h>
#include <ev.h> #include <ev.h>
/* TODO: update doxygen, especially with mid arguments */
/* TODO: should this library encapsulate the Mercury initialization steps? /* TODO: should this library encapsulate the Mercury initialization steps?
* Right now it does for caller simplicity, but there isn't any * Right now it does for caller simplicity, but there isn't any
* technical reason. Because it hides the initialization (and context * technical reason. Because it hides the initialization (and context
...@@ -21,31 +23,36 @@ ...@@ -21,31 +23,36 @@
* produced at init time. * produced at init time.
*/ */
struct margo_instance;
typedef struct margo_instance* margo_instance_id;
/** /**
* Initializes margo library, including initializing underlying libevfibers * Initializes margo library, including initializing underlying libevfibers
* and Mercury instances. * and Mercury instances.
* @param [in] listen flag indicating whether to accept inbound RPCs or not * @param [in] listen flag indicating whether to accept inbound RPCs or not
* @param [in] local_addr address to listen on if listen is set * @param [in] local_addr address to listen on if listen is set
* @returns 0 on success, -1 upon error * @returns margo instance id on success, NULL upon error
*/ */
int margo_init(na_bool_t listen, const char* local_addr); margo_instance_id margo_init(na_bool_t listen, const char* local_addr);
/** /**
* Shuts down margo library and its underlying evfibers and mercury resources * Shuts down margo library and its underlying evfibers and mercury resources
*/ */
void margo_finalize(void); void margo_finalize(margo_instance_id mid);
/** /**
* Retrieve the HG class for the running Mercury instance * Retrieve the HG class for the running Mercury instance
* @returns pointer on success, NULL upon error * @returns pointer on success, NULL upon error
*/ */
hg_class_t* margo_get_class(void); hg_class_t* margo_get_class(margo_instance_id mid);
/** /**
* Retrieve the ABT pool associated with the main caller (whoever invoked the * Retrieve the ABT pool associated with the main caller (whoever invoked the
* init function); this is where margo will execute RPC handlers. * init function); this is where margo will execute RPC handlers.
*/ */
ABT_pool* margo_get_main_pool(void); ABT_pool* margo_get_main_pool(margo_instance_id mid);
/** /**
* Lookup the Mercury/NA address associated with the given string * Lookup the Mercury/NA address associated with the given string
...@@ -53,7 +60,7 @@ ABT_pool* margo_get_main_pool(void); ...@@ -53,7 +60,7 @@ ABT_pool* margo_get_main_pool(void);
* @param [out] addr Mercury NA address for remote host * @param [out] addr Mercury NA address for remote host
* @returns 0 on success, na_return_t values on error * @returns 0 on success, na_return_t values on error
*/ */
na_return_t margo_addr_lookup(const char* name, na_addr_t* addr); na_return_t margo_addr_lookup(margo_instance_id mid, const char* name, na_addr_t* addr);
/** /**
* Creates a handle to refer to an RPC that will be issued * Creates a handle to refer to an RPC that will be issued
...@@ -62,7 +69,7 @@ na_return_t margo_addr_lookup(const char* name, na_addr_t* addr); ...@@ -62,7 +69,7 @@ na_return_t margo_addr_lookup(const char* name, na_addr_t* addr);
* @param [out] handle * @param [out] handle
* @returns 0 on success, hg_return_t values on error * @returns 0 on success, hg_return_t values on error
*/ */
hg_return_t margo_create_handle(na_addr_t addr, hg_id_t id, hg_return_t margo_create_handle(margo_instance_id mid, na_addr_t addr, hg_id_t id,
hg_handle_t *handle); hg_handle_t *handle);
/** /**
...@@ -72,6 +79,7 @@ hg_return_t margo_create_handle(na_addr_t addr, hg_id_t id, ...@@ -72,6 +79,7 @@ hg_return_t margo_create_handle(na_addr_t addr, hg_id_t id,
* @returns 0 on success, hg_return_t values on error * @returns 0 on success, hg_return_t values on error
*/ */
hg_return_t margo_forward( hg_return_t margo_forward(
margo_instance_id mid,
hg_handle_t handle, hg_handle_t handle,
void *in_struct); void *in_struct);
...@@ -88,6 +96,7 @@ hg_return_t margo_forward( ...@@ -88,6 +96,7 @@ hg_return_t margo_forward(
* @returns 0 on success, hg_return_t values on error * @returns 0 on success, hg_return_t values on error
*/ */
hg_return_t margo_bulk_transfer( hg_return_t margo_bulk_transfer(
margo_instance_id mid,
hg_bulk_context_t *context, hg_bulk_context_t *context,
hg_bulk_op_t op, hg_bulk_op_t op,
na_addr_t origin_addr, na_addr_t origin_addr,
...@@ -97,6 +106,11 @@ hg_return_t margo_bulk_transfer( ...@@ -97,6 +106,11 @@ hg_return_t margo_bulk_transfer(
size_t local_offset, size_t local_offset,
size_t size); size_t size);
/* given a particular hg_class, find the ABT pool that we intend to use to
* execute handlers.
*/
margo_instance_id margo_hg_class_to_instance(hg_class_t *class);
/** /**
* macro that defines a function to glue an RPC handler to a fiber * macro that defines a function to glue an RPC handler to a fiber
* @param [in] __name name of handler function * @param [in] __name name of handler function
...@@ -105,10 +119,14 @@ hg_return_t margo_bulk_transfer( ...@@ -105,10 +119,14 @@ hg_return_t margo_bulk_transfer(
static hg_return_t __name##_handler(hg_handle_t handle) { \ static hg_return_t __name##_handler(hg_handle_t handle) { \
int __ret; \ int __ret; \
ABT_pool* __pool; \ ABT_pool* __pool; \
margo_instance_id __mid; \
struct hg_info *__hgi; \
hg_handle_t* __handle = malloc(sizeof(*__handle)); \ hg_handle_t* __handle = malloc(sizeof(*__handle)); \
if(!__handle) return(HG_NOMEM_ERROR); \ if(!__handle) return(HG_NOMEM_ERROR); \
*__handle = handle; \ *__handle = handle; \
__pool = margo_get_main_pool(); \ __hgi = HG_Get_info(handle); \
__mid = margo_hg_class_to_instance(__hgi->hg_class); \
__pool = margo_get_main_pool(__mid); \
__ret = ABT_thread_create(*__pool, __name, __handle, ABT_THREAD_ATTR_NULL, NULL); \ __ret = ABT_thread_create(*__pool, __name, __handle, ABT_THREAD_ATTR_NULL, NULL); \
if(__ret != 0) { \ if(__ret != 0) { \
return(HG_NOMEM_ERROR); \ return(HG_NOMEM_ERROR); \
......
...@@ -13,18 +13,32 @@ ...@@ -13,18 +13,32 @@
#include "margo.h" #include "margo.h"
static na_class_t *network_class = NULL; struct margo_instance
static na_context_t *na_context = NULL; {
static hg_context_t *hg_context = NULL; na_class_t *network_class;
static hg_class_t *hg_class = NULL; na_context_t *na_context;
hg_context_t *hg_context;
hg_class_t *hg_class;
ABT_thread hg_progress_tid;
int hg_progress_shutdown_flag;
ABT_pool main_pool;
ABT_pool engine_pool;
ABT_xstream engine_xstream;
int table_index;
};
struct margo_handler_mapping
{
hg_class_t *class;
margo_instance_id mid;
};
#define MAX_HANDLER_MAPPING 8
static int handler_mapping_table_size = 0;
static struct margo_handler_mapping handler_mapping_table[MAX_HANDLER_MAPPING] = {0};
static ABT_thread hg_progress_tid;
static int hg_progress_shutdown_flag = 0;
static void hg_progress_fn(void* foo); static void hg_progress_fn(void* foo);
static ABT_pool main_pool;
static ABT_pool engine_pool;
static ABT_xstream engine_xstream;
struct handler_entry struct handler_entry
{ {
...@@ -33,40 +47,52 @@ struct handler_entry ...@@ -33,40 +47,52 @@ struct handler_entry
struct handler_entry *next; struct handler_entry *next;
}; };
int margo_init(na_bool_t listen, const char* local_addr) margo_instance_id margo_init(na_bool_t listen, const char* local_addr)
{ {
int ret; int ret;
ABT_xstream xstream; ABT_xstream xstream;
struct margo_instance *mid;
if(handler_mapping_table_size >= MAX_HANDLER_MAPPING)
return(NULL);
mid = malloc(sizeof(*mid));
if(!mid)
return(NULL);
memset(mid, 0, sizeof(*mid));
/* boilerplate HG initialization steps */ /* boilerplate HG initialization steps */
network_class = NA_Initialize(local_addr, listen); mid->network_class = NA_Initialize(local_addr, listen);
if(!network_class) if(!mid->network_class)
{ {
return(-1); free(mid);
return(NULL);
} }
na_context = NA_Context_create(network_class); mid->na_context = NA_Context_create(mid->network_class);
if(!na_context) if(!mid->na_context)
{ {
NA_Finalize(network_class); NA_Finalize(mid->network_class);
return(-1); free(mid);
return(NULL);
} }
hg_class = HG_Init(network_class, na_context, NULL); mid->hg_class = HG_Init(mid->network_class, mid->na_context, NULL);
if(!hg_class) if(!mid->hg_class)
{ {
NA_Context_destroy(network_class, na_context); NA_Context_destroy(mid->network_class, mid->na_context);
NA_Finalize(network_class); NA_Finalize(mid->network_class);
return(-1); free(mid);
return(NULL);
} }
hg_context = HG_Context_create(hg_class); mid->hg_context = HG_Context_create(mid->hg_class);
if(!hg_context) if(!mid->hg_context)
{ {
HG_Finalize(hg_class); HG_Finalize(mid->hg_class);
NA_Context_destroy(network_class, na_context); NA_Context_destroy(mid->network_class, mid->na_context);
NA_Finalize(network_class); NA_Finalize(mid->network_class);
return(-1); return(NULL);
} }
/* get the primary pool for the caller, this is where we will run ULTs to /* get the primary pool for the caller, this is where we will run ULTs to
...@@ -77,52 +103,65 @@ int margo_init(na_bool_t listen, const char* local_addr) ...@@ -77,52 +103,65 @@ int margo_init(na_bool_t listen, const char* local_addr)
{ {
/* TODO: err handling */ /* TODO: err handling */
fprintf(stderr, "Error: ABT_xstream_self()\n"); fprintf(stderr, "Error: ABT_xstream_self()\n");
return(-1); return(NULL);
} }
ret = ABT_xstream_get_main_pools(xstream, 1, &main_pool); ret = ABT_xstream_get_main_pools(xstream, 1, &mid->main_pool);
if(ret != 0) if(ret != 0)
{ {
/* TODO: err handling */ /* TODO: err handling */
fprintf(stderr, "Error: ABT_xstream_get_main_pools()\n"); fprintf(stderr, "Error: ABT_xstream_get_main_pools()\n");
return(-1); return(NULL);
} }
/* create an ES and ULT to drive Mercury progress */ /* create an ES and ULT to drive Mercury progress */
ret = ABT_snoozer_xstream_create(1, &engine_pool, &engine_xstream); ret = ABT_snoozer_xstream_create(1, &mid->engine_pool, &mid->engine_xstream);
if(ret != 0) if(ret != 0)
{ {
/* TODO: err handling */ /* TODO: err handling */
fprintf(stderr, "Error: ABT_snoozer_xstream_create()\n"); fprintf(stderr, "Error: ABT_snoozer_xstream_create()\n");
return(-1); return(NULL);
} }
ret = ABT_thread_create(engine_pool, hg_progress_fn, NULL, ret = ABT_thread_create(mid->engine_pool, hg_progress_fn, mid,
ABT_THREAD_ATTR_NULL, &hg_progress_tid); ABT_THREAD_ATTR_NULL, &mid->hg_progress_tid);
if(ret != 0) if(ret != 0)
{ {
/* TODO: err handling */ /* TODO: err handling */
fprintf(stderr, "Error: ABT_thread_create()\n"); fprintf(stderr, "Error: ABT_thread_create()\n");
return(-1); return(NULL);
} }
return 0; handler_mapping_table[handler_mapping_table_size].mid = mid;
handler_mapping_table[handler_mapping_table_size].class = mid->hg_class;
mid->table_index = handler_mapping_table_size;
handler_mapping_table_size++;
return mid;
} }
void margo_finalize(void) void margo_finalize(margo_instance_id mid)
{ {