ssg-rpc.c 9.71 KB
Newer Older
1 2 3 4 5 6 7 8 9
/*
 * Copyright (c) 2016 UChicago Argonne, LLC
 *
 * See COPYRIGHT in top-level directory.
 */

#include "ssg-config.h"

#include <stdlib.h>
10
#include <assert.h>
11 12 13 14 15 16 17 18

#include <mercury.h>
#include <abt.h>
#include <margo.h>

#include "ssg.h"
#include "ssg-internal.h"

19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38
#define SSG_VIEW_BUF_DEF_SIZE (128 * 1024)

/* SSG RPC types and (de)serialization routines */

/* NOTE: keep in sync with ssg_group_descriptor_t definition in ssg-internal.h */
MERCURY_GEN_STRUCT_PROC(ssg_group_descriptor_t, \
    ((uint64_t)     (magic_nr)) \
    ((uint64_t)     (name_hash)) \
    ((hg_string_t)  (addr_str)));

MERCURY_GEN_PROC(ssg_group_attach_request_t, \
    ((ssg_group_descriptor_t)   (group_descriptor))
    ((hg_bulk_t)                (bulk_handle)));

MERCURY_GEN_PROC(ssg_group_attach_response_t, \
    ((hg_string_t)  (group_name)) \
    ((uint32_t)     (group_size)) \
    ((hg_size_t)    (view_buf_size)));

/* SSG RPC handler prototypes */
39 40
DECLARE_MARGO_RPC_HANDLER(ssg_group_attach_recv_ult)

41 42 43
/* internal helper routine prototypes */
static int ssg_group_view_serialize(
    ssg_group_view_t *view, void **buf, hg_size_t *buf_size);
Shane Snyder's avatar
Shane Snyder committed
44

Shane Snyder's avatar
Shane Snyder committed
45
/* SSG RPC IDs */
46 47 48 49 50 51 52 53 54 55 56 57 58 59
static hg_id_t ssg_group_attach_rpc_id;

/* ssg_register_rpcs
 *
 *
 */
void ssg_register_rpcs()
{
    hg_class_t *hgcl = NULL;

    hgcl = margo_get_class(ssg_inst->mid);
    if (!hgcl) return;

    /* register HG RPCs for SSG */
Shane Snyder's avatar
Shane Snyder committed
60
    ssg_group_attach_rpc_id = MERCURY_REGISTER(hgcl, "ssg_group_attach",
61 62
        ssg_group_attach_request_t, ssg_group_attach_response_t,
        ssg_group_attach_recv_ult_handler);
63 64 65 66 67 68 69 70

    return;
}

/* ssg_group_attach_send
 *
 *
 */
71 72 73 74 75
int ssg_group_attach_send(
    ssg_group_descriptor_t * group_descriptor,
    char ** group_name,
    int * group_size,
    void ** view_buf)
76 77 78 79
{
    hg_class_t *hgcl = NULL;
    hg_addr_t member_addr = HG_ADDR_NULL;
    hg_handle_t handle = HG_HANDLE_NULL;
80 81 82 83 84
    hg_bulk_t bulk_handle = HG_BULK_NULL;
    void *tmp_view_buf = NULL, *b;
    hg_size_t tmp_view_buf_size = SSG_VIEW_BUF_DEF_SIZE;
    ssg_group_attach_request_t attach_req;
    ssg_group_attach_response_t attach_resp;
85
    hg_return_t hret;
86 87 88 89 90
    int sret = SSG_FAILURE;

    *group_name = NULL;
    *group_size = 0;
    *view_buf = NULL;
91 92 93 94 95

    hgcl = margo_get_class(ssg_inst->mid);
    if (!hgcl) goto fini;

    /* lookup the address of the given group member */
Shane Snyder's avatar
Shane Snyder committed
96 97
    hret = margo_addr_lookup(ssg_inst->mid, group_descriptor->addr_str,
        &member_addr);
98 99 100 101 102 103
    if (hret != HG_SUCCESS) goto fini;

    hret = HG_Create(margo_get_context(ssg_inst->mid), member_addr,
        ssg_group_attach_rpc_id, &handle);
    if (hret != HG_SUCCESS) goto fini;

104 105 106 107 108 109 110 111 112 113 114 115
    /* allocate a buffer of the given size to try to store the group view in */
    /* NOTE: We don't know if this buffer is big enough to store the complete
     * view. If the buffers is not large enough, the group member we are
     * attaching too will send a NACK indicating the necessary buffer size
     */
    tmp_view_buf = malloc(tmp_view_buf_size);
    if (!tmp_view_buf) goto fini;

    hret = HG_Bulk_create(hgcl, 1, &tmp_view_buf, &tmp_view_buf_size,
        HG_BULK_WRITE_ONLY, &bulk_handle);
    if (hret != HG_SUCCESS) goto fini;

116
    /* send an attach request to the given group member address */
117 118 119
    memcpy(&attach_req.group_descriptor, group_descriptor, sizeof(*group_descriptor));
    attach_req.bulk_handle = bulk_handle;
    hret = margo_forward(ssg_inst->mid, handle, &attach_req);
120 121
    if (hret != HG_SUCCESS) goto fini;

122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171
    hret = HG_Get_output(handle, &attach_resp);
    if (hret != HG_SUCCESS) goto fini;

    /* if our initial buffer is too small, reallocate to the exact size & reattach */
    if (attach_resp.view_buf_size > tmp_view_buf_size)
    {
        b = realloc(tmp_view_buf, attach_resp.view_buf_size);
        if(!b)
        {
            HG_Free_output(handle, &attach_resp);
            goto fini;
        }
        tmp_view_buf = b;
        tmp_view_buf_size = attach_resp.view_buf_size;

        /* free old bulk handle and recreate it */
        HG_Bulk_free(bulk_handle);
        hret = HG_Bulk_create(hgcl, 1, &tmp_view_buf, &tmp_view_buf_size,
            HG_BULK_WRITE_ONLY, &bulk_handle);
        if (hret != HG_SUCCESS) goto fini;

        attach_req.bulk_handle = bulk_handle;
        hret = margo_forward(ssg_inst->mid, handle, &attach_req);
        if (hret != HG_SUCCESS) goto fini;

        HG_Free_output(handle, &attach_resp);
        hret = HG_Get_output(handle, &attach_resp);
        if (hret != HG_SUCCESS) goto fini;
    }

    /* readjust view buf size if initial guess was too large */
    if (attach_resp.view_buf_size < tmp_view_buf_size)
    {
        b = realloc(tmp_view_buf, attach_resp.view_buf_size);
        if(!b)
        {
            HG_Free_output(handle, &attach_resp);
            goto fini;
        }
        tmp_view_buf = b;
    }

    /* set output pointers according to the returned view parameters */
    *group_name = strdup(attach_resp.group_name);
    *group_size = (int)attach_resp.group_size;
    *view_buf = tmp_view_buf;

    HG_Free_output(handle, &attach_resp);
    tmp_view_buf = NULL;
    sret = SSG_SUCCESS;
172 173 174
fini:
    if (hgcl && member_addr != HG_ADDR_NULL) HG_Addr_free(hgcl, member_addr);
    if (handle != HG_HANDLE_NULL) HG_Destroy(handle);
175 176
    if (bulk_handle != HG_BULK_NULL) HG_Bulk_free(bulk_handle);
    free(tmp_view_buf);
177

178
    return sret;
179 180
}

181 182
static void ssg_group_attach_recv_ult(
    hg_handle_t handle)
183
{
184 185
    hg_class_t *hgcl = NULL;
    const struct hg_info *hgi = NULL;
Shane Snyder's avatar
Shane Snyder committed
186
    ssg_group_t *g = NULL;
187 188 189 190 191 192 193
    ssg_group_attach_request_t attach_req;
    ssg_group_attach_response_t attach_resp;
    hg_size_t view_size_requested;
    void *view_buf = NULL;
    hg_size_t view_buf_size;
    hg_bulk_t bulk_handle = HG_BULK_NULL;
    int sret;
Shane Snyder's avatar
Shane Snyder committed
194 195 196 197
    hg_return_t hret;

    if (!ssg_inst) goto fini;

198 199 200 201 202 203
    hgcl = margo_get_class(ssg_inst->mid);
    if (!hgcl) goto fini;

    hgi = HG_Get_info(handle);
    if (!hgi) goto fini;
    hret = HG_Get_input(handle, &attach_req);
Shane Snyder's avatar
Shane Snyder committed
204
    if (hret != HG_SUCCESS) goto fini;
205
    view_size_requested = HG_Bulk_get_size(attach_req.bulk_handle);
Shane Snyder's avatar
Shane Snyder committed
206 207

    /* look for the given group in my local table of groups */
208
    HASH_FIND(hh, ssg_inst->group_table, &attach_req.group_descriptor.name_hash,
Shane Snyder's avatar
Shane Snyder committed
209 210 211
        sizeof(uint64_t), g);
    if (!g)
    {
212
        HG_Free_input(handle, &attach_req);
Shane Snyder's avatar
Shane Snyder committed
213 214 215
        goto fini;
    }

216 217 218 219 220 221
    sret = ssg_group_view_serialize(&g->view, &view_buf, &view_buf_size);
    if (sret != SSG_SUCCESS)
    {
        HG_Free_input(handle, &attach_req);
        goto fini;
    }
Shane Snyder's avatar
Shane Snyder committed
222

223 224 225 226 227 228 229 230 231 232
    if (view_size_requested >= view_buf_size)
    {
        /* if attacher's buf is large enough, transfer the view */
        hret = HG_Bulk_create(hgcl, 1, &view_buf, &view_buf_size, HG_BULK_READ_ONLY,
            &bulk_handle);
        if (hret != HG_SUCCESS)
        {
            HG_Free_input(handle, &attach_req);
            goto fini;
        }
Shane Snyder's avatar
Shane Snyder committed
233

234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249
        hret = margo_bulk_transfer(ssg_inst->mid, HG_BULK_PUSH, hgi->addr,
            attach_req.bulk_handle, 0, bulk_handle, 0, view_buf_size);
        if (hret != HG_SUCCESS)
        {
            HG_Free_input(handle, &attach_req);
            goto fini;
        }
    }

    /* set the response and send back */
    attach_resp.group_name = g->name;
    attach_resp.group_size = (int)g->view.size;
    attach_resp.view_buf_size = view_buf_size;
    margo_respond(ssg_inst->mid, handle, &attach_resp);

    HG_Free_input(handle, &attach_req);
Shane Snyder's avatar
Shane Snyder committed
250
fini:
251 252 253 254
    free(view_buf); /* TODO: cache this */
    if (handle != HG_HANDLE_NULL) HG_Destroy(handle);
    if (bulk_handle != HG_BULK_NULL) HG_Bulk_free(bulk_handle);

255 256 257
    return;
}
DEFINE_MARGO_RPC_HANDLER(ssg_group_attach_recv_ult)
Shane Snyder's avatar
Shane Snyder committed
258

259 260 261 262 263 264 265 266 267 268
static int ssg_group_view_serialize(
    ssg_group_view_t *view, void **buf, hg_size_t *buf_size)
{
    unsigned int i;
    hg_size_t view_size = 0;
    int tmp_size;
    void *view_buf, *p;

    *buf = NULL;
    *buf_size = 0;
Shane Snyder's avatar
Shane Snyder committed
269

270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297
    /* first determine view size */
    for (i = 0; i < view->size; i++)
    {
        view_size += (strlen(view->member_states[i].addr_str) + 1);
    }

    view_buf = malloc(view_size);
    if(!view_buf)
        return SSG_FAILURE;

    p = view_buf;
    for (i = 0; i < view->size; i++)
    {
        tmp_size = strlen(view->member_states[i].addr_str) + 1;
        memcpy(p, view->member_states[i].addr_str, tmp_size);
        p += tmp_size;
    }

    *buf = view_buf;
    *buf_size = view_size;

    return SSG_SUCCESS;
}

/* custom SSG RPC proc routines */

hg_return_t hg_proc_ssg_group_id_t(
    hg_proc_t proc, void *data)
Shane Snyder's avatar
Shane Snyder committed
298
{
299
    ssg_group_descriptor_t **group_descriptor = (ssg_group_descriptor_t **)data;
Shane Snyder's avatar
Shane Snyder committed
300 301 302 303 304
    hg_return_t hret = HG_PROTOCOL_ERROR;

    switch(hg_proc_get_op(proc))
    {
        case HG_ENCODE:
305 306 307 308 309 310
            hret = hg_proc_ssg_group_descriptor_t(proc, *group_descriptor);
            if (hret != HG_SUCCESS)
            {
                hret = HG_PROTOCOL_ERROR;
                return hret;
            }
Shane Snyder's avatar
Shane Snyder committed
311 312
            break;
        case HG_DECODE:
313 314 315 316 317 318
            *group_descriptor = malloc(sizeof(**group_descriptor));
            if (!(*group_descriptor))
            {
                hret = HG_NOMEM_ERROR;
                return hret;
            }
319
            memset(*group_descriptor, 0, sizeof(**group_descriptor));
320 321 322 323 324 325
            hret = hg_proc_ssg_group_descriptor_t(proc, *group_descriptor);
            if (hret != HG_SUCCESS)
            {
                hret = HG_PROTOCOL_ERROR;
                return hret;
            }
Shane Snyder's avatar
Shane Snyder committed
326
            (*group_descriptor)->ref_count = 1;
Shane Snyder's avatar
Shane Snyder committed
327 328
            break;
        case HG_FREE:
Shane Snyder's avatar
Shane Snyder committed
329
            if((*group_descriptor)->ref_count == 1)
330
            {
Shane Snyder's avatar
Shane Snyder committed
331 332 333 334 335 336
                free((*group_descriptor)->addr_str);
                free(*group_descriptor);
            }
            else
            {
                (*group_descriptor)->ref_count--;
337 338
            }
            hret = HG_SUCCESS;
Shane Snyder's avatar
Shane Snyder committed
339 340 341 342 343 344 345
            break;
        default:
            break;
    }

    return hret;
}