margo.h 8.97 KB
Newer Older
1 2 3 4 5 6 7 8 9
/*
 * (C) 2015 The University of Chicago
 * 
 * See COPYRIGHT in top-level directory.
 */

#ifndef __MARGO
#define __MARGO

10 11 12 13
#ifdef __cplusplus
extern "C" {
#endif

14 15 16 17 18
#include <mercury_bulk.h>
#include <mercury.h>
#include <mercury_macros.h>
#include <abt.h>

19 20 21
struct margo_instance;
typedef struct margo_instance* margo_instance_id;

22
#define MARGO_INSTANCE_NULL ((margo_instance_id)NULL)
23
#define MARGO_DEFAULT_MPLEX_ID 0
24

Jonathan Jenkins's avatar
Jonathan Jenkins committed
25 26 27 28 29

/**
 * Initializes margo library.
 * @param [in] use_progress_thread Boolean flag to use a dedicated thread for
 *                                 running Mercury's progress loop. If false,
30
 *                                 it will run in the caller's thread context.
Jonathan Jenkins's avatar
Jonathan Jenkins committed
31 32
 * @param [in] rpc_thread_count    Number of threads to use for running RPC
 *                                 calls. A value of 0 directs Margo to execute
33 34 35 36 37 38
 *                                 RPCs in the caller's thread context.
 *                                 Clients (i.e processes that will *not* 
 *                                 service incoming RPCs) should use a value 
 *                                 of 0. A value of -1 directs Margo to use 
 *                                 the same execution context as that used 
 *                                 for Mercury progress.
Jonathan Jenkins's avatar
Jonathan Jenkins committed
39 40
 * @param [in] hg_context
 * @returns margo instance id on success, MARGO_INSTANCE_NULL upon error
41 42 43 44 45
 *
 * NOTE: Servers (processes expecting to service incoming RPC requests) must
 * specify non-zero values for use_progress_thread and rpc_thread_count *or*
 * call margo_wait_for_finalize() after margo_init() to relinguish control to 
 * Margo.
Jonathan Jenkins's avatar
Jonathan Jenkins committed
46 47 48 49
 */
margo_instance_id margo_init(int use_progress_thread, int rpc_thread_count,
    hg_context_t *hg_context);

50
/**
Jonathan Jenkins's avatar
doc fix  
Jonathan Jenkins committed
51
 * Initializes margo library from given argobots and Mercury instances.
Philip Carns's avatar
Philip Carns committed
52 53 54
 * @param [in] progress_pool Argobots pool to drive communication
 * @param [in] handler_pool Argobots pool to service RPC handlers
 * @param [in] hg_context Mercury context
Jonathan Jenkins's avatar
fixupme  
Jonathan Jenkins committed
55
 * @returns margo instance id on success, MARGO_INSTANCE_NULL upon error
56
 */
Jonathan Jenkins's avatar
Jonathan Jenkins committed
57
margo_instance_id margo_init_pool(ABT_pool progress_pool, ABT_pool handler_pool,
Jonathan Jenkins's avatar
Jonathan Jenkins committed
58
    hg_context_t *hg_context);
59 60

/**
Philip Carns's avatar
Philip Carns committed
61
 * Shuts down margo library and its underlying abt and mercury resources
Philip Carns's avatar
Philip Carns committed
62
 * @param [in] mid Margo instance
63
 */
64
void margo_finalize(margo_instance_id mid);
65 66 67 68 69 70 71 72 73 74 75 76

/**
 * Suspends the caller until some other entity (e.g. an RPC, thread, or
 * signal handler) invokes margo_finalize().
 *
 * NOTE: This informs Margo that the calling thread no longer needs to be 
 * scheduled for execution if it is sharing an Argobots pool with the
 * progress engine.
 *
 * @param [in] mid Margo instance
 */
void margo_wait_for_finalize(margo_instance_id mid);
77 78

/**
Philip Carns's avatar
Philip Carns committed
79 80 81
 * Retrieve the abt_handler pool that was associated with the instance at 
 *    initialization time
 * @param [in] mid Margo instance
82
 */
83
ABT_pool* margo_get_handler_pool(margo_instance_id mid);
84

85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100
/**
 * Retrieve the Mercury context that was associated with this instance at
 *    initialization time
 * @param [in] mid Margo instance
 * @return the Mercury context used in margo_init
 */
hg_context_t* margo_get_context(margo_instance_id mid);

/**
 * Retrieve the Mercury class that was associated with this instance at
 *    initialization time
 * @param [in] mid Margo instance
 * @return the Mercury class used in margo_init
 */
hg_class_t* margo_get_class(margo_instance_id mid);

101 102
/**
 * Forward an RPC request to a remote host
Philip Carns's avatar
Philip Carns committed
103
 * @param [in] mid Margo instance
104 105 106 107 108
 * @param [in] handle identifier for the RPC to be sent
 * @param [in] in_struct input argument struct for RPC
 * @returns 0 on success, hg_return_t values on error
 */
hg_return_t margo_forward(
109
    margo_instance_id mid,
110 111 112
    hg_handle_t handle,
    void *in_struct);

113 114 115 116 117 118 119 120 121 122 123 124 125 126
/**
 * Forward an RPC request to a remote host with a user-defined timeout
 * @param [in] mid Margo instance
 * @param [in] handle identifier for the RPC to be sent
 * @param [in] in_struct input argument struct for RPC
 * @param [in] timeout_ms timeout in milliseconds
 * @returns 0 on success, hg_return_t values on error
 */
hg_return_t margo_forward_timed(
    margo_instance_id mid,
    hg_handle_t handle,
    void *in_struct,
    double timeout_ms);

Jonathan Jenkins's avatar
Jonathan Jenkins committed
127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144
/**
 * Send an RPC response, waiting for completion before returning
 * control to the calling ULT.
 * Note: this call is typically not needed as RPC listeners need not concern
 * themselves with what happens after an RPC finishes. However, there are cases
 * when this is useful (deferring resource cleanup, calling margo_finalize()
 * for e.g. a shutdown RPC).
 * @param [in] mid Margo instance
 * @param [in] handle identifier for the RPC for which a response is being
 * sent
 * @param [in] out_struct output argument struct for response
 * @return HG_SUCCESS on success, hg_return_t values on error. See HG_Respond.
 */
hg_return_t margo_respond(
    margo_instance_id mid,
    hg_handle_t handle,
    void *out_struct);

145 146
/** 
 * Perform a bulk transfer
Philip Carns's avatar
Philip Carns committed
147
 * @param [in] mid Margo instance
148 149 150 151 152 153 154 155 156 157
 * @param [in] op type of operation to perform
 * @param [in] origin_addr remote Mercury address
 * @param [in] origin_handle remote Mercury bulk memory handle
 * @param [in] origin_offset offset into remote bulk memory to access
 * @param [in] local_handle local bulk memory handle
 * @param [in] local_offset offset into local bulk memory to access
 * @param [in] size size (in bytes) of transfer
 * @returns 0 on success, hg_return_t values on error
 */
hg_return_t margo_bulk_transfer(
158
    margo_instance_id mid,
159
    hg_bulk_op_t op,
160
    hg_addr_t origin_addr,
161 162 163 164 165 166
    hg_bulk_t origin_handle,
    size_t origin_offset,
    hg_bulk_t local_handle,
    size_t local_offset,
    size_t size);

167 168 169
/**
 * address lookup
 * @param [in] name             lookup name
Jonathan Jenkins's avatar
fixupme  
Jonathan Jenkins committed
170 171
 * @param [out] addr            return address
 * @returns HG_SUCCESS on success
172
 */
173
hg_return_t margo_addr_lookup(
174
    margo_instance_id mid,
Philip Carns's avatar
Philip Carns committed
175
    const char   *name,
176
    hg_addr_t    *addr);
177

178 179
/**
 * Suspends the calling ULT for a specified time duration
180
 * @param [in] mid Margo instance
181 182 183
 * @param [in] timeout_ms timeout duration in milliseconds
 */
void margo_thread_sleep(
184
    margo_instance_id mid,
185 186
    double timeout_ms);

Philip Carns's avatar
Philip Carns committed
187 188
/**
 * Retrive the Margo instance that has been associated with a Mercury class
189
 * @param [in] cl Mercury class
Philip Carns's avatar
Philip Carns committed
190
 * @returns Margo instance on success, NULL on error
191
 */
192
margo_instance_id margo_hg_class_to_instance(hg_class_t *cl);
193

Philip Carns's avatar
Philip Carns committed
194 195 196 197
/** 
 * Registers an RPC with margo that is associated with a multiplexed service
 * @param [in] mid Margo instance
 * @param [in] id Mercury RPC identifier
Philip Carns's avatar
Philip Carns committed
198
 * @param [in] mplex_id multiplexing identifier
Philip Carns's avatar
Philip Carns committed
199 200 201 202
 * @param [in] pool Argobots pool the handler will execute in
 */
int margo_register_mplex(margo_instance_id mid, hg_id_t id, uint32_t mplex_id, ABT_pool pool);

Philip Carns's avatar
Philip Carns committed
203 204 205 206 207 208 209 210 211
/**
 * Maps an RPC id and mplex id to the pool that it should execute on
 * @param [in] mid Margo instance
 * @param [in] id Mercury RPC identifier
 * @param [in] mplex_id multiplexing identifier
 * @param [out] pool Argobots pool the handler will execute in
 */
int margo_lookup_mplex(margo_instance_id mid, hg_id_t id, uint32_t mplex_id, ABT_pool *pool);

Philip Carns's avatar
Philip Carns committed
212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229
/**
 * Enables diagnostic collection on specified Margo instance
 *
 * @param [in] mid Margo instance
 * @returns void
 */
void margo_diag_start(margo_instance_id mid);

/**
 * Appends diagnostic statistics (enabled via margo_diag_start()) to specified 
 * output file.
 *
 * @param [in] mid Margo instance
 * @param [in] file output file ("-" for stdout)
 * @returns void
 */
void margo_diag_dump(margo_instance_id mid, const char* file);

230 231 232 233 234 235 236 237 238 239
#define MARGO_REGISTER(__mid, __func_name, __in_t, __out_t, __handler, __mplex_id, __pool) do { \
    hg_return_t __hret; \
    hg_id_t __id; \
    hg_bool_t __flag; \
    int __ret; \
    __hret = HG_Registered_name(margo_get_class(__mid), __func_name, &__id, &__flag); \
    assert(__hret == HG_SUCCESS); \
    if(!__flag) \
        __id = MERCURY_REGISTER(margo_get_class(__mid), __func_name, __in_t, __out_t, __handler); \
    __ret = margo_register_mplex(__mid, __id, __mplex_id, __pool); \
Philip Carns's avatar
typo  
Philip Carns committed
240
    assert(__ret == 0); \
241 242
} while(0)

243
/**
Philip Carns's avatar
Philip Carns committed
244
 * macro that defines a function to glue an RPC handler to a ult handler
245 246
 * @param [in] __name name of handler function
 */
247
#define DEFINE_MARGO_RPC_HANDLER(__name) \
248
hg_return_t __name##_handler(hg_handle_t handle) { \
249
    int __ret; \
Philip Carns's avatar
Philip Carns committed
250
    ABT_pool __pool; \
251
    margo_instance_id __mid; \
Shane Snyder's avatar
Shane Snyder committed
252
    const struct hg_info *__hgi; \
253 254
    __hgi = HG_Get_info(handle); \
    __mid = margo_hg_class_to_instance(__hgi->hg_class); \
255
    __ret = margo_lookup_mplex(__mid, __hgi->id, __hgi->target_id, (&__pool)); \
Philip Carns's avatar
Philip Carns committed
256 257 258 259
    if(__ret != 0) { \
        return(HG_INVALID_PARAM); \
    }\
    __ret = ABT_thread_create(__pool, (void (*)(void *))__name, handle, ABT_THREAD_ATTR_NULL, NULL); \
260 261 262 263 264 265
    if(__ret != 0) { \
        return(HG_NOMEM_ERROR); \
    } \
    return(HG_SUCCESS); \
}

Philip Carns's avatar
Philip Carns committed
266 267
/**
 * macro that declares the prototype for a function to glue an RPC 
Philip Carns's avatar
Philip Carns committed
268
 * handler to a ult
Philip Carns's avatar
Philip Carns committed
269 270
 * @param [in] __name name of handler function
 */
271
#define DECLARE_MARGO_RPC_HANDLER(__name) hg_return_t __name##_handler(hg_handle_t handle);
272

273 274 275 276
#ifdef __cplusplus
}
#endif

277
#endif /* __MARGO */