ssg-rpc.c 5.47 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
/*
 * Copyright (c) 2016 UChicago Argonne, LLC
 *
 * See COPYRIGHT in top-level directory.
 */

#include "ssg-config.h"

#include <stdlib.h>

#include <mercury.h>
#include <abt.h>
#include <margo.h>

#include "ssg.h"
#include "ssg-internal.h"

Shane Snyder's avatar
Shane Snyder committed
18
/* SSG RPCS handler prototypes */
19
20
21
static void ssg_lookup_ult(void * arg);
DECLARE_MARGO_RPC_HANDLER(ssg_group_attach_recv_ult)

Shane Snyder's avatar
Shane Snyder committed
22
23
24
/* SSG RPC (de)serialization routine prototypes */
static hg_return_t hg_proc_ssg_group_id_t(hg_proc_t proc, void *data);

25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
/* SSG RPC ids */
static hg_id_t ssg_group_attach_rpc_id;

/* ssg_register_rpcs
 *
 *
 */
void ssg_register_rpcs()
{
    hg_class_t *hgcl = NULL;

    hgcl = margo_get_class(ssg_inst->mid);
    if (!hgcl) return;

    /* register HG RPCs for SSG */
Shane Snyder's avatar
Shane Snyder committed
40
41
    ssg_group_attach_rpc_id = MERCURY_REGISTER(hgcl, "ssg_group_attach",
        ssg_group_descriptor_t, void, ssg_group_attach_recv_ult_handler);
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63

    return;
}

/* ssg_group_lookup
 *
 * 
 */
struct lookup_ult_args
{
    ssg_group_t *g; 
    ssg_member_id_t member_id;
    const char *addr_str;
    hg_return_t out;
};

hg_return_t ssg_group_lookup(
    ssg_group_t * g,
    const char * const addr_strs[])
{
    ABT_thread *ults;
    struct lookup_ult_args *args;
Shane Snyder's avatar
Shane Snyder committed
64
    unsigned int i, r;
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
    int aret;
    hg_return_t hret = HG_SUCCESS;

    if (g == NULL) return HG_INVALID_PARAM;

    /* initialize ULTs */
    ults = malloc(g->group_view.size * sizeof(*ults));
    if (ults == NULL) return HG_NOMEM_ERROR;
    args = malloc(g->group_view.size * sizeof(*args));
    if (args == NULL)
    {
        free(ults);
        return HG_NOMEM_ERROR;
    }
    for (i = 0; i < g->group_view.size; i++)
        ults[i] = ABT_THREAD_NULL;
    
    for (i = 1; i < g->group_view.size; i++)
    {
        r = (g->self_id + i) % g->group_view.size;
        args[r].g = g;
        args[r].member_id = r;
        args[r].addr_str = addr_strs[r];
        aret = ABT_thread_create(*margo_get_handler_pool(ssg_inst->mid),
                &ssg_lookup_ult, &args[r], ABT_THREAD_ATTR_NULL, &ults[r]);
        if (aret != ABT_SUCCESS) {
            hret = HG_OTHER_ERROR;
            goto fini;
        }
    }

    /* wait on all */
    for (i = 1; i < g->group_view.size; i++)
    {
        r = (g->self_id + i) % g->group_view.size;
        aret = ABT_thread_join(ults[r]);
        ABT_thread_free(&ults[r]); 
        ults[r] = ABT_THREAD_NULL; // in case of cascading failure from join
        if (aret != ABT_SUCCESS)
        {   
            hret = HG_OTHER_ERROR;
            break;
        }
        else if (args[r].out != HG_SUCCESS)
        {
            fprintf(stderr, "Error: SSG unable to lookup HG address for rank %d"
                "(err=%d)\n", r, args[r].out);
            hret = args[r].out;
            break;
        }
    }

fini:
    /* cleanup */ 
    for (i = 0; i < g->group_view.size; i++)
    {
        if (ults[i] != ABT_THREAD_NULL)
        {
            ABT_thread_cancel(ults[i]);
            ABT_thread_free(ults[i]);
        }
    }
    free(ults);
    free(args);

    return hret;
}

static void ssg_lookup_ult(
    void * arg)
{
    struct lookup_ult_args *l = arg;
    ssg_group_t *g = l->g;

    l->out = margo_addr_lookup(ssg_inst->mid, l->addr_str,
        &g->group_view.member_states[l->member_id].addr);
    return;
}

/* ssg_group_attach_send
 *
 *
 */
Shane Snyder's avatar
Shane Snyder committed
148
hg_return_t ssg_group_attach_send(ssg_group_descriptor_t * group_descriptor)
149
150
151
152
153
154
155
156
157
158
{
    hg_class_t *hgcl = NULL;
    hg_addr_t member_addr = HG_ADDR_NULL;
    hg_handle_t handle = HG_HANDLE_NULL;
    hg_return_t hret;

    hgcl = margo_get_class(ssg_inst->mid);
    if (!hgcl) goto fini;

    /* lookup the address of the given group member */
Shane Snyder's avatar
Shane Snyder committed
159
160
    hret = margo_addr_lookup(ssg_inst->mid, group_descriptor->addr_str,
        &member_addr);
161
162
163
164
165
166
167
    if (hret != HG_SUCCESS) goto fini;

    hret = HG_Create(margo_get_context(ssg_inst->mid), member_addr,
        ssg_group_attach_rpc_id, &handle);
    if (hret != HG_SUCCESS) goto fini;

    /* send an attach request to the given group member address */
Shane Snyder's avatar
Shane Snyder committed
168
    hret = margo_forward(ssg_inst->mid, handle, group_descriptor);
169
170
171
172
173
174
175
176
177
178
179
180
    if (hret != HG_SUCCESS) goto fini;

    /* TODO: hold on to leader addr so we don't have to look it up again? */
fini:
    if (hgcl && member_addr != HG_ADDR_NULL) HG_Addr_free(hgcl, member_addr);
    if (handle != HG_HANDLE_NULL) HG_Destroy(handle);

    return hret;
}

static void ssg_group_attach_recv_ult(hg_handle_t handle)
{
Shane Snyder's avatar
Shane Snyder committed
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
    ssg_group_t *g = NULL;
    ssg_group_descriptor_t group_descriptor;
    hg_return_t hret;

    /* TODO: how to handle errors */
    if (!ssg_inst) goto fini;

    hret = HG_Get_input(handle, &group_descriptor);
    if (hret != HG_SUCCESS) goto fini;

    /* look for the given group in my local table of groups */
    HASH_FIND(hh, ssg_inst->group_table, &group_descriptor.name_hash,
        sizeof(uint64_t), g);
    if (!g)
    {
        HG_Free_input(handle, &group_descriptor);
        goto fini;
    }

    margo_respond(ssg_inst->mid, handle, NULL);

    HG_Free_input(handle, &group_descriptor);

fini:
205
206
207
208
    HG_Destroy(handle);
    return;
}
DEFINE_MARGO_RPC_HANDLER(ssg_group_attach_recv_ult)
Shane Snyder's avatar
Shane Snyder committed
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232

/* SSG RPC (de)serialization routines */

#if 0
static hg_return_t hg_proc_ssg_group_id_t(hg_proc_t proc, void *data)
{
    ssg_group_descriptor_t *group_descriptor = (ssg_group_descriptor_t *)data;
    hg_return_t hret = HG_PROTOCOL_ERROR;

    switch(hg_proc_get_op(proc))
    {
        case HG_ENCODE:
            break;
        case HG_DECODE:
            break;
        case HG_FREE:
            break;
        default:
            break;
    }

    return hret;
}
#endif