gm_init.c 9.51 KB
Newer Older
1
2
3
4
5
6
/* -*- Mode: C; c-basic-offset:4 ; -*- */
/*
 *  (C) 2006 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

7
8
9
10
11
12
13
14
15
16
17
18
19
#include "gm_impl.h"

MPID_nem_netmod_funcs_t MPIDI_nem_gm_funcs = {
    MPID_nem_gm_init,
    MPID_nem_gm_finalize,
    MPID_nem_gm_ckpt_shutdown,
    MPID_nem_gm_poll,
    MPID_nem_gm_send,
    MPID_nem_gm_get_business_card,
    MPID_nem_gm_connect_to_root,
    MPID_nem_gm_vc_init,
    MPID_nem_gm_vc_destroy,
    MPID_nem_gm_vc_terminate
20
21
22
};


23
24
25
26
27
28
29
30
31
32
33
34
35
#define MAX_GM_BOARDS 16
#define UNIQUE_ID_LEN 6
#define MPIDI_CH3I_PORT_KEY "port"
#define MPIDI_CH3I_UNIQUE_KEY "unique"
#define UNDEFINED_UNIQUE_ID_VAL "\0\0\0\0\0\0"

static unsigned char unique_id[UNIQUE_ID_LEN] = UNDEFINED_UNIQUE_ID_VAL;
static int port_id;

int MPID_nem_module_gm_num_send_tokens = 0;

struct gm_port *MPID_nem_module_gm_port = 0;

36
37
MPID_nem_queue_ptr_t MPID_nem_process_free_queue = NULL;

38
#define FREE_SEND_QUEUE_ELEMENTS MPID_NEM_NUM_CELLS
39
40
MPID_nem_gm_send_queue_head_t MPID_nem_gm_send_queue = {0};
MPID_nem_gm_send_queue_t *MPID_nem_gm_send_free_queue = 0;
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99

static int my_pg_rank;

#undef FUNCNAME
#define FUNCNAME init_gm
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
static int
init_gm (int *boardId, int *portId, unsigned char unique_id[])
{
    int mpi_errno = MPI_SUCCESS;
    gm_status_t status;
    int max_gm_ports;
    
    status = gm_init();
    MPIU_ERR_CHKANDJUMP1 (status != GM_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**gm_init", "**gm_init %d", status);
    
    max_gm_ports = gm_num_ports (NULL);
    
    for (*portId = 0; *portId < max_gm_ports; ++*portId)
    {
	/* skip reserved gm ports */
	if (*portId == 0 || *portId == 1 || *portId == 3)
	    continue;
	for (*boardId = 0; *boardId < MAX_GM_BOARDS; ++*boardId)
	{
	    status = gm_open (&MPID_nem_module_gm_port, *boardId, *portId, " ", GM_API_VERSION);
		
	    switch (status)
	    {
	    case GM_SUCCESS:
		/* successfuly allocated a port */
                status = gm_get_unique_board_id (MPID_nem_module_gm_port, (char *)unique_id);
                MPIU_ERR_CHKANDJUMP1 (status != GM_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**gm_get_unique_board_id", "**gm_get_unique_board_id %d", status);
                goto fn_exit;
		break;
	    case GM_INCOMPATIBLE_LIB_AND_DRIVER:
                MPIU_ERR_SETANDJUMP (mpi_errno, MPI_ERR_OTHER, "**gm_incompatible_lib");
		break;
	    default:
		break;
	    }
		
	}
    }
    
    /* if we got here, we tried all ports and couldn't find a free one */
    MPIU_ERR_SETANDJUMP (mpi_errno, MPI_ERR_OTHER, "**gm_no_port");

 fn_exit:
    return mpi_errno;
 fn_fail:
    memset (unique_id, 0, UNIQUE_ID_LEN);
    goto fn_exit;
}


/*
   int  
100
   MPID_nem_gm_init(MPID_nem_queue_ptr_t proc_recv_queue, MPID_nem_queue_ptr_t proc_free_queue, MPID_nem_cell_ptr_t proc_elements, int num_proc_elements,
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
	          MPID_nem_cell_ptr_t module_elements, int num_module_elements,
		  MPID_nem_queue_ptr_t *module_free_queue)

   IN
       proc_recv_queue -- main recv queue for the process
       proc_free_queue -- main free queueu for the process
       proc_elements -- pointer to the process' queue elements
       num_proc_elements -- number of process' queue elements
       module_elements -- pointer to queue elements to be used by this module
       num_module_elements -- number of queue elements for this module
   OUT
       free_queue -- pointer to the free queue for this module.  The process will return elements to
                     this queue
*/

#undef FUNCNAME
117
#define FUNCNAME MPID_nem_gm_init
118
119
120
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int
121
MPID_nem_gm_init (MPID_nem_queue_ptr_t proc_recv_queue, 
122
123
124
                         MPID_nem_queue_ptr_t proc_free_queue, 
                         MPID_nem_cell_ptr_t proc_elements,   int num_proc_elements,
                         MPID_nem_cell_ptr_t module_elements, int num_module_elements, 
125
                         MPID_nem_queue_ptr_t *module_free_queue,
126
127
128
129
130
131
132
133
134
135
                         MPIDI_PG_t *pg_p, int pg_rank,
                         char **bc_val_p, int *val_max_sz_p)
{
    int mpi_errno = MPI_SUCCESS;
    int board_id;
    int ret;
    gm_status_t status;
    int i;

    /* first make sure that our private fields in the vc fit into the area provided  */
136
    MPIU_Assert(sizeof(MPID_nem_gm_vc_area) <= MPID_NEM_VC_NETMOD_AREA_LEN);
137
138

    my_pg_rank = pg_rank;
139

140
141
142
    mpi_errno = init_gm(&board_id, &port_id, unique_id);
    if (mpi_errno) MPIU_ERR_POP(mpi_errno);

143
    mpi_errno = MPID_nem_gm_get_business_card(pg_rank, bc_val_p, val_max_sz_p);
144
145
    if (mpi_errno) MPIU_ERR_POP(mpi_errno);

146
147
148
149
150
    MPID_nem_process_free_queue = proc_free_queue;

    status = gm_register_memory (MPID_nem_module_gm_port, (void *)proc_elements, sizeof (MPID_nem_cell_t) * num_proc_elements);
    MPIU_ERR_CHKANDJUMP1 (status != GM_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**gm_regmem", "**gm_regmem %d", status);

151
152
    MPID_nem_module_gm_num_send_tokens = gm_num_send_tokens(MPID_nem_module_gm_port);

153
    mpi_errno = MPID_nem_gm_recv_init();
154
155
    if (mpi_errno) MPIU_ERR_POP(mpi_errno);

156
157
    MPID_nem_gm_send_queue.head = NULL;
    MPID_nem_gm_send_queue.tail = NULL;
158

159
    MPID_nem_gm_send_free_queue = NULL;
160
161
162
    
    for (i = 0; i < FREE_SEND_QUEUE_ELEMENTS; ++i)
    {
163
	MPID_nem_gm_send_queue_t *e;
164
	
165
166
167
168
	e = MPIU_Malloc (sizeof (MPID_nem_gm_send_queue_t));
        if (e == NULL) MPIU_CHKMEM_SETERR (mpi_errno, sizeof (MPID_nem_gm_send_queue_t), "gm module send queue");
	e->next = MPID_nem_gm_send_free_queue;
	MPID_nem_gm_send_free_queue = e;
169
170
    }
    
171
    mpi_errno = MPID_nem_gm_lmt_init();
172
173
174
175
176
177
178
179
180
181
182
    if (mpi_errno) MPIU_ERR_POP (mpi_errno);

    *module_free_queue = NULL;

 fn_exit:
    return mpi_errno;
 fn_fail:
    goto fn_exit;
}

#undef FUNCNAME
183
#define FUNCNAME MPID_nem_gm_get_business_card
184
185
186
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int
187
MPID_nem_gm_get_business_card (int my_rank, char **bc_val_p, int *val_max_sz_p)
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
{
    int mpi_errno = MPI_SUCCESS;

    mpi_errno = MPIU_Str_add_int_arg (bc_val_p, val_max_sz_p, MPIDI_CH3I_PORT_KEY, port_id);
    if (mpi_errno != MPIU_STR_SUCCESS)
    {
	if (mpi_errno == MPIU_STR_NOMEM) {
	    MPIU_ERR_SET(mpi_errno, MPI_ERR_OTHER, "**buscard_len");
	}
	else {
	    MPIU_ERR_SET(mpi_errno, MPI_ERR_OTHER, "**buscard");
	}
	return mpi_errno;
    }

    mpi_errno = MPIU_Str_add_binary_arg (bc_val_p, val_max_sz_p, MPIDI_CH3I_UNIQUE_KEY, (char *)unique_id, UNIQUE_ID_LEN);
    if (mpi_errno != MPIU_STR_SUCCESS)
    {
	if (mpi_errno == MPIU_STR_NOMEM) {
	    MPIU_ERR_SET(mpi_errno, MPI_ERR_OTHER, "**buscard_len");
	}
	else {
	    MPIU_ERR_SET(mpi_errno, MPI_ERR_OTHER, "**buscard");
	}
	return mpi_errno;
    }

    return mpi_errno;
}

#undef FUNCNAME
219
#define FUNCNAME MPID_nem_gm_get_port_unique_from_bc
220
221
222
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int
223
MPID_nem_gm_get_port_unique_from_bc (const char *business_card, unsigned *port_id, unsigned char *unique_id)
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
{
    int mpi_errno = MPI_SUCCESS;
    int len;
    int tmp_port_id;
    
    mpi_errno = MPIU_Str_get_int_arg (business_card, MPIDI_CH3I_PORT_KEY, &tmp_port_id);
    if (mpi_errno != MPIU_STR_SUCCESS) {
	/* FIXME: create a real error string for this */
	MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER, "**argstr_hostd");
    }
    *port_id = (unsigned)tmp_port_id;

    mpi_errno = MPIU_Str_get_binary_arg (business_card, MPIDI_CH3I_UNIQUE_KEY, (char *)unique_id, UNIQUE_ID_LEN, &len);
    if (mpi_errno != MPIU_STR_SUCCESS || len != UNIQUE_ID_LEN) {
	/* FIXME: create a real error string for this */
	MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER, "**argstr_hostd");
    }

 fn_exit:
    return mpi_errno;
 fn_fail:
    goto fn_exit;
}

#undef FUNCNAME
249
#define FUNCNAME MPID_nem_gm_connect_to_root
250
251
252
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int
253
MPID_nem_gm_connect_to_root (const char *business_card, MPIDI_VC_t *new_vc)
254
255
256
257
258
259
{
    /* In GM, once the VC is initialized there's nothing extra that we need to do to establish a connection */
    return MPI_SUCCESS;
}

#undef FUNCNAME
260
#define FUNCNAME MPID_nem_gm_vc_init
261
262
263
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int
264
MPID_nem_gm_vc_init(MPIDI_VC_t *vc)
265
{    
266
267
    int mpi_errno = MPI_SUCCESS;
    int ret;
268
    char *business_card;
269
    int pmi_errno;
270
271
    int val_max_sz;
    MPIU_CHKLMEM_DECL(1);
272
273

    /* Allocate space for the business card */
274
275
276
    pmi_errno = PMI_KVS_Get_value_length_max(&val_max_sz);
    MPIU_ERR_CHKANDJUMP1(pmi_errno, mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %d", pmi_errno);
    MPIU_CHKLMEM_MALLOC(business_card, char *, val_max_sz, mpi_errno, "bc");
277

278
    mpi_errno = vc->pg->getConnInfo(vc->pg_rank, business_card, val_max_sz, vc->pg);
279
    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
280
281
282

    VC_FIELD(vc, source_id) = my_pg_rank; /* FIXME: this is only valid for processes in COMM_WORLD */

283
    mpi_errno = MPID_nem_gm_get_port_unique_from_bc (business_card, &VC_FIELD(vc, gm_port_id), VC_FIELD(vc, gm_unique_id));
284
285
286
287
288
289
290
291
292
293
294
    if (mpi_errno) MPIU_ERR_POP (mpi_errno);

    ret = gm_unique_id_to_node_id (MPID_nem_module_gm_port, (char *)VC_FIELD(vc, gm_unique_id), &VC_FIELD(vc, gm_node_id));
    /* --BEGIN ERROR HANDLING-- */
    if (ret != GM_SUCCESS)
    {
	mpi_errno = MPI_ERR_INTERN;
	goto fn_fail;
    }
    /* --END ERROR HANDLING-- */

295
296
fn_exit:
    MPIU_CHKLMEM_FREEALL();
297
    return mpi_errno;
298
299
fn_fail:
    goto fn_exit;
300
301
302
}

#undef FUNCNAME
303
#define FUNCNAME MPID_nem_gm_vc_destroy
304
305
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
306
int MPID_nem_gm_vc_destroy(MPIDI_VC_t *vc)
307
308
{
    int mpi_errno = MPI_SUCCESS;   
309
310
311
312
 fn_exit:   
    return mpi_errno;
 fn_fail:
    goto fn_exit;
313
314
315
}

#undef FUNCNAME
316
#define FUNCNAME MPID_nem_gm_vc_terminate
317
318
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
319
int MPID_nem_gm_vc_terminate (MPIDI_VC_t *vc)
320
321
322
{
    return MPI_SUCCESS;
}