gm_init.c 9.49 KB
Newer Older
1
2
3
4
5
6
/* -*- Mode: C; c-basic-offset:4 ; -*- */
/*
 *  (C) 2006 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

7
8
9
10
11
12
13
14
15
16
17
#include "gm_impl.h"

MPID_nem_netmod_funcs_t MPIDI_nem_gm_funcs = {
    MPID_nem_gm_init,
    MPID_nem_gm_finalize,
    MPID_nem_gm_ckpt_shutdown,
    MPID_nem_gm_poll,
    MPID_nem_gm_get_business_card,
    MPID_nem_gm_connect_to_root,
    MPID_nem_gm_vc_init,
    MPID_nem_gm_vc_destroy,
18
19
    MPID_nem_gm_vc_terminate,
    NULL /* anysource iprobe */
20
21
22
};


23
24
25
26
27
28
29
30
31
32
33
34
35
#define MAX_GM_BOARDS 16
#define UNIQUE_ID_LEN 6
#define MPIDI_CH3I_PORT_KEY "port"
#define MPIDI_CH3I_UNIQUE_KEY "unique"
#define UNDEFINED_UNIQUE_ID_VAL "\0\0\0\0\0\0"

static unsigned char unique_id[UNIQUE_ID_LEN] = UNDEFINED_UNIQUE_ID_VAL;
static int port_id;

int MPID_nem_module_gm_num_send_tokens = 0;

struct gm_port *MPID_nem_module_gm_port = 0;

36
37
MPID_nem_queue_ptr_t MPID_nem_process_free_queue = NULL;

38
#define FREE_SEND_QUEUE_ELEMENTS MPID_NEM_NUM_CELLS
39
40
MPID_nem_gm_send_queue_head_t MPID_nem_gm_send_queue = {0};
MPID_nem_gm_send_queue_t *MPID_nem_gm_send_free_queue = 0;
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99

static int my_pg_rank;

#undef FUNCNAME
#define FUNCNAME init_gm
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
static int
init_gm (int *boardId, int *portId, unsigned char unique_id[])
{
    int mpi_errno = MPI_SUCCESS;
    gm_status_t status;
    int max_gm_ports;
    
    status = gm_init();
    MPIU_ERR_CHKANDJUMP1 (status != GM_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**gm_init", "**gm_init %d", status);
    
    max_gm_ports = gm_num_ports (NULL);
    
    for (*portId = 0; *portId < max_gm_ports; ++*portId)
    {
	/* skip reserved gm ports */
	if (*portId == 0 || *portId == 1 || *portId == 3)
	    continue;
	for (*boardId = 0; *boardId < MAX_GM_BOARDS; ++*boardId)
	{
	    status = gm_open (&MPID_nem_module_gm_port, *boardId, *portId, " ", GM_API_VERSION);
		
	    switch (status)
	    {
	    case GM_SUCCESS:
		/* successfuly allocated a port */
                status = gm_get_unique_board_id (MPID_nem_module_gm_port, (char *)unique_id);
                MPIU_ERR_CHKANDJUMP1 (status != GM_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**gm_get_unique_board_id", "**gm_get_unique_board_id %d", status);
                goto fn_exit;
		break;
	    case GM_INCOMPATIBLE_LIB_AND_DRIVER:
                MPIU_ERR_SETANDJUMP (mpi_errno, MPI_ERR_OTHER, "**gm_incompatible_lib");
		break;
	    default:
		break;
	    }
		
	}
    }
    
    /* if we got here, we tried all ports and couldn't find a free one */
    MPIU_ERR_SETANDJUMP (mpi_errno, MPI_ERR_OTHER, "**gm_no_port");

 fn_exit:
    return mpi_errno;
 fn_fail:
    memset (unique_id, 0, UNIQUE_ID_LEN);
    goto fn_exit;
}


/*
   int  
100
   MPID_nem_gm_init(MPID_nem_queue_ptr_t proc_recv_queue, MPID_nem_queue_ptr_t proc_free_queue, MPID_nem_cell_ptr_t proc_elements, int num_proc_elements,
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
	          MPID_nem_cell_ptr_t module_elements, int num_module_elements,
		  MPID_nem_queue_ptr_t *module_free_queue)

   IN
       proc_recv_queue -- main recv queue for the process
       proc_free_queue -- main free queueu for the process
       proc_elements -- pointer to the process' queue elements
       num_proc_elements -- number of process' queue elements
       module_elements -- pointer to queue elements to be used by this module
       num_module_elements -- number of queue elements for this module
   OUT
       free_queue -- pointer to the free queue for this module.  The process will return elements to
                     this queue
*/

#undef FUNCNAME
117
#define FUNCNAME MPID_nem_gm_init
118
119
120
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int
121
MPID_nem_gm_init (MPID_nem_queue_ptr_t proc_recv_queue, 
122
123
124
                         MPID_nem_queue_ptr_t proc_free_queue, 
                         MPID_nem_cell_ptr_t proc_elements,   int num_proc_elements,
                         MPID_nem_cell_ptr_t module_elements, int num_module_elements, 
125
                         MPID_nem_queue_ptr_t *module_free_queue,
126
127
128
129
130
131
132
133
134
135
                         MPIDI_PG_t *pg_p, int pg_rank,
                         char **bc_val_p, int *val_max_sz_p)
{
    int mpi_errno = MPI_SUCCESS;
    int board_id;
    int ret;
    gm_status_t status;
    int i;

    /* first make sure that our private fields in the vc fit into the area provided  */
136
    MPIU_Assert(sizeof(MPID_nem_gm_vc_area) <= MPID_NEM_VC_NETMOD_AREA_LEN);
137
138

    my_pg_rank = pg_rank;
139

140
141
142
    mpi_errno = init_gm(&board_id, &port_id, unique_id);
    if (mpi_errno) MPIU_ERR_POP(mpi_errno);

143
    mpi_errno = MPID_nem_gm_get_business_card(pg_rank, bc_val_p, val_max_sz_p);
144
145
    if (mpi_errno) MPIU_ERR_POP(mpi_errno);

146
147
148
149
150
    MPID_nem_process_free_queue = proc_free_queue;

    status = gm_register_memory (MPID_nem_module_gm_port, (void *)proc_elements, sizeof (MPID_nem_cell_t) * num_proc_elements);
    MPIU_ERR_CHKANDJUMP1 (status != GM_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**gm_regmem", "**gm_regmem %d", status);

151
152
    MPID_nem_module_gm_num_send_tokens = gm_num_send_tokens(MPID_nem_module_gm_port);

153
    mpi_errno = MPID_nem_gm_recv_init();
154
155
    if (mpi_errno) MPIU_ERR_POP(mpi_errno);

156
157
    MPID_nem_gm_send_queue.head = NULL;
    MPID_nem_gm_send_queue.tail = NULL;
158

159
    MPID_nem_gm_send_free_queue = NULL;
160
161
162
    
    for (i = 0; i < FREE_SEND_QUEUE_ELEMENTS; ++i)
    {
163
	MPID_nem_gm_send_queue_t *e;
164
	
165
166
167
168
	e = MPIU_Malloc (sizeof (MPID_nem_gm_send_queue_t));
        if (e == NULL) MPIU_CHKMEM_SETERR (mpi_errno, sizeof (MPID_nem_gm_send_queue_t), "gm module send queue");
	e->next = MPID_nem_gm_send_free_queue;
	MPID_nem_gm_send_free_queue = e;
169
170
    }
    
171
    mpi_errno = MPID_nem_gm_lmt_init();
172
173
174
175
176
177
178
179
180
181
182
    if (mpi_errno) MPIU_ERR_POP (mpi_errno);

    *module_free_queue = NULL;

 fn_exit:
    return mpi_errno;
 fn_fail:
    goto fn_exit;
}

#undef FUNCNAME
183
#define FUNCNAME MPID_nem_gm_get_business_card
184
185
186
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int
187
MPID_nem_gm_get_business_card (int my_rank, char **bc_val_p, int *val_max_sz_p)
188
189
{
    int mpi_errno = MPI_SUCCESS;
190
    int str_errno = MPIU_STR_SUCCESS;
191

192
193
194
195
    str_errno = MPIU_Str_add_int_arg (bc_val_p, val_max_sz_p, MPIDI_CH3I_PORT_KEY, port_id);
    if (str_errno) {
        MPIU_ERR_CHKANDJUMP(str_errno == MPIU_STR_NOMEM, mpi_errno, MPI_ERR_OTHER, "**buscard_len");
        MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**buscard");
196
197
    }

198
199
200
201
    str_errno = MPIU_Str_add_binary_arg (bc_val_p, val_max_sz_p, MPIDI_CH3I_UNIQUE_KEY, (char *)unique_id, UNIQUE_ID_LEN);
    if (str_errno) {
        MPIU_ERR_CHKANDJUMP(str_errno == MPIU_STR_NOMEM, mpi_errno, MPI_ERR_OTHER, "**buscard_len");
        MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**buscard");
202
203
    }

204
 fn_exit:
205
    return mpi_errno;
206
207
 fn_fail:
    goto fn_exit;
208
209
210
}

#undef FUNCNAME
211
#define FUNCNAME MPID_nem_gm_get_port_unique_from_bc
212
213
214
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int
215
MPID_nem_gm_get_port_unique_from_bc (const char *business_card, unsigned *port_id, unsigned char *unique_id)
216
217
{
    int mpi_errno = MPI_SUCCESS;
218
    int str_errno = MPIU_STR_SUCCESS;
219
220
221
    int len;
    int tmp_port_id;
    
222
223
224
    str_errno = MPIU_Str_get_int_arg (business_card, MPIDI_CH3I_PORT_KEY, &tmp_port_id);
    /* FIXME: create a real error string for this */
    MPIU_ERR_CHKANDJUMP(str_errno, mpi_errno, MPI_ERR_OTHER, "**argstr_hostd");
225
226
    *port_id = (unsigned)tmp_port_id;

227
228
229
    str_errno = MPIU_Str_get_binary_arg (business_card, MPIDI_CH3I_UNIQUE_KEY, (char *)unique_id, UNIQUE_ID_LEN, &len);
    /* FIXME: create a real error string for this */
    MPIU_ERR_CHKANDJUMP(str_errno || len != UNIQUE_ID_LEN, mpi_errno, MPI_ERR_OTHER, "**argstr_hostd");
230
231
232
233
234
235
236
237

 fn_exit:
    return mpi_errno;
 fn_fail:
    goto fn_exit;
}

#undef FUNCNAME
238
#define FUNCNAME MPID_nem_gm_connect_to_root
239
240
241
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int
242
MPID_nem_gm_connect_to_root (const char *business_card, MPIDI_VC_t *new_vc)
243
244
245
246
247
248
{
    /* In GM, once the VC is initialized there's nothing extra that we need to do to establish a connection */
    return MPI_SUCCESS;
}

#undef FUNCNAME
249
#define FUNCNAME MPID_nem_gm_vc_init
250
251
252
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int
253
MPID_nem_gm_vc_init(MPIDI_VC_t *vc)
254
{    
255
256
    int mpi_errno = MPI_SUCCESS;
    int ret;
257
    char *business_card;
258
    int pmi_errno;
259
260
    int val_max_sz;
    MPIU_CHKLMEM_DECL(1);
261
262

    /* Allocate space for the business card */
263
264
265
    pmi_errno = PMI_KVS_Get_value_length_max(&val_max_sz);
    MPIU_ERR_CHKANDJUMP1(pmi_errno, mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %d", pmi_errno);
    MPIU_CHKLMEM_MALLOC(business_card, char *, val_max_sz, mpi_errno, "bc");
266

267
    mpi_errno = vc->pg->getConnInfo(vc->pg_rank, business_card, val_max_sz, vc->pg);
268
    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
269
270
271

    VC_FIELD(vc, source_id) = my_pg_rank; /* FIXME: this is only valid for processes in COMM_WORLD */

272
    mpi_errno = MPID_nem_gm_get_port_unique_from_bc (business_card, &VC_FIELD(vc, gm_port_id), VC_FIELD(vc, gm_unique_id));
273
274
275
276
277
278
279
280
281
282
283
    if (mpi_errno) MPIU_ERR_POP (mpi_errno);

    ret = gm_unique_id_to_node_id (MPID_nem_module_gm_port, (char *)VC_FIELD(vc, gm_unique_id), &VC_FIELD(vc, gm_node_id));
    /* --BEGIN ERROR HANDLING-- */
    if (ret != GM_SUCCESS)
    {
	mpi_errno = MPI_ERR_INTERN;
	goto fn_fail;
    }
    /* --END ERROR HANDLING-- */

284
285
fn_exit:
    MPIU_CHKLMEM_FREEALL();
286
    return mpi_errno;
287
288
fn_fail:
    goto fn_exit;
289
290
291
}

#undef FUNCNAME
292
#define FUNCNAME MPID_nem_gm_vc_destroy
293
294
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
295
int MPID_nem_gm_vc_destroy(MPIDI_VC_t *vc)
296
297
{
    int mpi_errno = MPI_SUCCESS;   
298
299
300
301
 fn_exit:   
    return mpi_errno;
 fn_fail:
    goto fn_exit;
302
303
304
}

#undef FUNCNAME
305
#define FUNCNAME MPID_nem_gm_vc_terminate
306
307
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
308
int MPID_nem_gm_vc_terminate (MPIDI_VC_t *vc)
309
310
311
{
    return MPI_SUCCESS;
}