ssg-rpc.c 9.93 KB
Newer Older
1 2 3 4 5 6 7 8 9
/*
 * Copyright (c) 2016 UChicago Argonne, LLC
 *
 * See COPYRIGHT in top-level directory.
 */

#include "ssg-config.h"

#include <stdlib.h>
10
#include <assert.h>
11 12 13 14 15 16 17 18

#include <mercury.h>
#include <abt.h>
#include <margo.h>

#include "ssg.h"
#include "ssg-internal.h"

19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38
#define SSG_VIEW_BUF_DEF_SIZE (128 * 1024)

/* SSG RPC types and (de)serialization routines */

/* NOTE: keep in sync with ssg_group_descriptor_t definition in ssg-internal.h */
MERCURY_GEN_STRUCT_PROC(ssg_group_descriptor_t, \
    ((uint64_t)     (magic_nr)) \
    ((uint64_t)     (name_hash)) \
    ((hg_string_t)  (addr_str)));

MERCURY_GEN_PROC(ssg_group_attach_request_t, \
    ((ssg_group_descriptor_t)   (group_descriptor))
    ((hg_bulk_t)                (bulk_handle)));

MERCURY_GEN_PROC(ssg_group_attach_response_t, \
    ((hg_string_t)  (group_name)) \
    ((uint32_t)     (group_size)) \
    ((hg_size_t)    (view_buf_size)));

/* SSG RPC handler prototypes */
39 40
DECLARE_MARGO_RPC_HANDLER(ssg_group_attach_recv_ult)

41 42 43
/* internal helper routine prototypes */
static int ssg_group_view_serialize(
    ssg_group_view_t *view, void **buf, hg_size_t *buf_size);
Shane Snyder's avatar
Shane Snyder committed
44

Shane Snyder's avatar
Shane Snyder committed
45
/* SSG RPC IDs */
46 47 48 49 50 51 52 53 54
static hg_id_t ssg_group_attach_rpc_id;

/* ssg_register_rpcs
 *
 *
 */
void ssg_register_rpcs()
{
    /* register HG RPCs for SSG */
55
    MARGO_REGISTER(ssg_inst->mid, "ssg_group_attach",
56
        ssg_group_attach_request_t, ssg_group_attach_response_t,
57
        ssg_group_attach_recv_ult, &ssg_group_attach_rpc_id);
58 59 60 61 62 63 64 65

    return;
}

/* ssg_group_attach_send
 *
 *
 */
66 67 68 69 70
int ssg_group_attach_send(
    ssg_group_descriptor_t * group_descriptor,
    char ** group_name,
    int * group_size,
    void ** view_buf)
71 72 73 74
{
    hg_class_t *hgcl = NULL;
    hg_addr_t member_addr = HG_ADDR_NULL;
    hg_handle_t handle = HG_HANDLE_NULL;
75 76 77 78 79
    hg_bulk_t bulk_handle = HG_BULK_NULL;
    void *tmp_view_buf = NULL, *b;
    hg_size_t tmp_view_buf_size = SSG_VIEW_BUF_DEF_SIZE;
    ssg_group_attach_request_t attach_req;
    ssg_group_attach_response_t attach_resp;
80
    hg_return_t hret;
81 82 83 84 85
    int sret = SSG_FAILURE;

    *group_name = NULL;
    *group_size = 0;
    *view_buf = NULL;
86 87 88 89 90

    hgcl = margo_get_class(ssg_inst->mid);
    if (!hgcl) goto fini;

    /* lookup the address of the given group member */
Shane Snyder's avatar
Shane Snyder committed
91 92
    hret = margo_addr_lookup(ssg_inst->mid, group_descriptor->addr_str,
        &member_addr);
93 94 95 96 97 98
    if (hret != HG_SUCCESS) goto fini;

    hret = HG_Create(margo_get_context(ssg_inst->mid), member_addr,
        ssg_group_attach_rpc_id, &handle);
    if (hret != HG_SUCCESS) goto fini;

99 100 101 102 103 104 105 106 107 108 109 110
    /* allocate a buffer of the given size to try to store the group view in */
    /* NOTE: We don't know if this buffer is big enough to store the complete
     * view. If the buffers is not large enough, the group member we are
     * attaching too will send a NACK indicating the necessary buffer size
     */
    tmp_view_buf = malloc(tmp_view_buf_size);
    if (!tmp_view_buf) goto fini;

    hret = HG_Bulk_create(hgcl, 1, &tmp_view_buf, &tmp_view_buf_size,
        HG_BULK_WRITE_ONLY, &bulk_handle);
    if (hret != HG_SUCCESS) goto fini;

111
    /* send an attach request to the given group member address */
112 113 114
    memcpy(&attach_req.group_descriptor, group_descriptor, sizeof(*group_descriptor));
    attach_req.bulk_handle = bulk_handle;
    hret = margo_forward(ssg_inst->mid, handle, &attach_req);
115 116
    if (hret != HG_SUCCESS) goto fini;

117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166
    hret = HG_Get_output(handle, &attach_resp);
    if (hret != HG_SUCCESS) goto fini;

    /* if our initial buffer is too small, reallocate to the exact size & reattach */
    if (attach_resp.view_buf_size > tmp_view_buf_size)
    {
        b = realloc(tmp_view_buf, attach_resp.view_buf_size);
        if(!b)
        {
            HG_Free_output(handle, &attach_resp);
            goto fini;
        }
        tmp_view_buf = b;
        tmp_view_buf_size = attach_resp.view_buf_size;

        /* free old bulk handle and recreate it */
        HG_Bulk_free(bulk_handle);
        hret = HG_Bulk_create(hgcl, 1, &tmp_view_buf, &tmp_view_buf_size,
            HG_BULK_WRITE_ONLY, &bulk_handle);
        if (hret != HG_SUCCESS) goto fini;

        attach_req.bulk_handle = bulk_handle;
        hret = margo_forward(ssg_inst->mid, handle, &attach_req);
        if (hret != HG_SUCCESS) goto fini;

        HG_Free_output(handle, &attach_resp);
        hret = HG_Get_output(handle, &attach_resp);
        if (hret != HG_SUCCESS) goto fini;
    }

    /* readjust view buf size if initial guess was too large */
    if (attach_resp.view_buf_size < tmp_view_buf_size)
    {
        b = realloc(tmp_view_buf, attach_resp.view_buf_size);
        if(!b)
        {
            HG_Free_output(handle, &attach_resp);
            goto fini;
        }
        tmp_view_buf = b;
    }

    /* set output pointers according to the returned view parameters */
    *group_name = strdup(attach_resp.group_name);
    *group_size = (int)attach_resp.group_size;
    *view_buf = tmp_view_buf;

    HG_Free_output(handle, &attach_resp);
    tmp_view_buf = NULL;
    sret = SSG_SUCCESS;
167 168 169
fini:
    if (hgcl && member_addr != HG_ADDR_NULL) HG_Addr_free(hgcl, member_addr);
    if (handle != HG_HANDLE_NULL) HG_Destroy(handle);
170 171
    if (bulk_handle != HG_BULK_NULL) HG_Bulk_free(bulk_handle);
    free(tmp_view_buf);
172

173
    return sret;
174 175
}

176 177
static void ssg_group_attach_recv_ult(
    hg_handle_t handle)
178
{
179 180
    hg_class_t *hgcl = NULL;
    const struct hg_info *hgi = NULL;
Shane Snyder's avatar
Shane Snyder committed
181
    ssg_group_t *g = NULL;
182 183 184 185 186 187 188
    ssg_group_attach_request_t attach_req;
    ssg_group_attach_response_t attach_resp;
    hg_size_t view_size_requested;
    void *view_buf = NULL;
    hg_size_t view_buf_size;
    hg_bulk_t bulk_handle = HG_BULK_NULL;
    int sret;
Shane Snyder's avatar
Shane Snyder committed
189 190 191 192
    hg_return_t hret;

    if (!ssg_inst) goto fini;

193 194 195 196 197 198
    hgcl = margo_get_class(ssg_inst->mid);
    if (!hgcl) goto fini;

    hgi = HG_Get_info(handle);
    if (!hgi) goto fini;
    hret = HG_Get_input(handle, &attach_req);
Shane Snyder's avatar
Shane Snyder committed
199
    if (hret != HG_SUCCESS) goto fini;
200
    view_size_requested = HG_Bulk_get_size(attach_req.bulk_handle);
Shane Snyder's avatar
Shane Snyder committed
201 202

    /* look for the given group in my local table of groups */
203
    HASH_FIND(hh, ssg_inst->group_table, &attach_req.group_descriptor.name_hash,
Shane Snyder's avatar
Shane Snyder committed
204 205 206
        sizeof(uint64_t), g);
    if (!g)
    {
207
        HG_Free_input(handle, &attach_req);
Shane Snyder's avatar
Shane Snyder committed
208 209 210
        goto fini;
    }

211 212 213 214 215 216
    sret = ssg_group_view_serialize(&g->view, &view_buf, &view_buf_size);
    if (sret != SSG_SUCCESS)
    {
        HG_Free_input(handle, &attach_req);
        goto fini;
    }
Shane Snyder's avatar
Shane Snyder committed
217

218 219 220 221 222 223 224 225 226 227
    if (view_size_requested >= view_buf_size)
    {
        /* if attacher's buf is large enough, transfer the view */
        hret = HG_Bulk_create(hgcl, 1, &view_buf, &view_buf_size, HG_BULK_READ_ONLY,
            &bulk_handle);
        if (hret != HG_SUCCESS)
        {
            HG_Free_input(handle, &attach_req);
            goto fini;
        }
Shane Snyder's avatar
Shane Snyder committed
228

229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244
        hret = margo_bulk_transfer(ssg_inst->mid, HG_BULK_PUSH, hgi->addr,
            attach_req.bulk_handle, 0, bulk_handle, 0, view_buf_size);
        if (hret != HG_SUCCESS)
        {
            HG_Free_input(handle, &attach_req);
            goto fini;
        }
    }

    /* set the response and send back */
    attach_resp.group_name = g->name;
    attach_resp.group_size = (int)g->view.size;
    attach_resp.view_buf_size = view_buf_size;
    margo_respond(ssg_inst->mid, handle, &attach_resp);

    HG_Free_input(handle, &attach_req);
Shane Snyder's avatar
Shane Snyder committed
245
fini:
246 247 248 249
    free(view_buf); /* TODO: cache this */
    if (handle != HG_HANDLE_NULL) HG_Destroy(handle);
    if (bulk_handle != HG_BULK_NULL) HG_Bulk_free(bulk_handle);

250 251 252
    return;
}
DEFINE_MARGO_RPC_HANDLER(ssg_group_attach_recv_ult)
Shane Snyder's avatar
Shane Snyder committed
253

254 255 256 257 258 259
static int ssg_group_view_serialize(
    ssg_group_view_t *view, void **buf, hg_size_t *buf_size)
{
    unsigned int i;
    hg_size_t view_size = 0;
    int tmp_size;
260
    void *view_buf, *buf_p, *str_p;
261 262 263

    *buf = NULL;
    *buf_size = 0;
Shane Snyder's avatar
Shane Snyder committed
264

265 266 267
    /* first determine view size */
    for (i = 0; i < view->size; i++)
    {
268 269 270 271
        if (view->member_states[i].addr_str)
            view_size += (strlen(view->member_states[i].addr_str) + 1);
        else
            view_size += 1;
272 273 274 275 276 277
    }

    view_buf = malloc(view_size);
    if(!view_buf)
        return SSG_FAILURE;

278
    buf_p = view_buf;
279 280
    for (i = 0; i < view->size; i++)
    {
281 282 283 284 285 286 287 288 289 290 291 292 293 294
        char null = '\0';

        if (view->member_states[i].addr_str)
        {
            tmp_size = strlen(view->member_states[i].addr_str) + 1;
            str_p = view->member_states[i].addr_str;
        }
        else
        {
            tmp_size = 1;
            str_p = &null;
        }
        memcpy(buf_p, str_p, tmp_size);
        buf_p += tmp_size;
295 296 297 298 299 300 301 302 303 304 305 306
    }

    *buf = view_buf;
    *buf_size = view_size;

    return SSG_SUCCESS;
}

/* custom SSG RPC proc routines */

hg_return_t hg_proc_ssg_group_id_t(
    hg_proc_t proc, void *data)
Shane Snyder's avatar
Shane Snyder committed
307
{
308
    ssg_group_descriptor_t **group_descriptor = (ssg_group_descriptor_t **)data;
Shane Snyder's avatar
Shane Snyder committed
309 310 311 312 313
    hg_return_t hret = HG_PROTOCOL_ERROR;

    switch(hg_proc_get_op(proc))
    {
        case HG_ENCODE:
314 315 316 317 318 319
            hret = hg_proc_ssg_group_descriptor_t(proc, *group_descriptor);
            if (hret != HG_SUCCESS)
            {
                hret = HG_PROTOCOL_ERROR;
                return hret;
            }
Shane Snyder's avatar
Shane Snyder committed
320 321
            break;
        case HG_DECODE:
322 323 324 325 326 327
            *group_descriptor = malloc(sizeof(**group_descriptor));
            if (!(*group_descriptor))
            {
                hret = HG_NOMEM_ERROR;
                return hret;
            }
328
            memset(*group_descriptor, 0, sizeof(**group_descriptor));
329 330 331 332 333 334
            hret = hg_proc_ssg_group_descriptor_t(proc, *group_descriptor);
            if (hret != HG_SUCCESS)
            {
                hret = HG_PROTOCOL_ERROR;
                return hret;
            }
Shane Snyder's avatar
Shane Snyder committed
335
            (*group_descriptor)->ref_count = 1;
Shane Snyder's avatar
Shane Snyder committed
336 337
            break;
        case HG_FREE:
Shane Snyder's avatar
Shane Snyder committed
338
            if((*group_descriptor)->ref_count == 1)
339
            {
Shane Snyder's avatar
Shane Snyder committed
340 341 342 343 344 345
                free((*group_descriptor)->addr_str);
                free(*group_descriptor);
            }
            else
            {
                (*group_descriptor)->ref_count--;
346 347
            }
            hret = HG_SUCCESS;
Shane Snyder's avatar
Shane Snyder committed
348 349 350 351 352 353 354
            break;
        default:
            break;
    }

    return hret;
}