mx_init.c 10.1 KB
Newer Older
1
2
3
4
5
6
7
/* -*- Mode: C; c-basic-offset:4 ; -*- */
/*
 *  (C) 2006 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

#include "mpid_nem_impl.h"
8
9
10
11
12
13
14
15
16
17
#include "mx_impl.h"

MPID_nem_netmod_funcs_t MPIDI_nem_mx_funcs = {
    MPID_nem_mx_init,
    MPID_nem_mx_finalize,
    MPID_nem_mx_poll,
    MPID_nem_mx_get_business_card,
    MPID_nem_mx_connect_to_root,
    MPID_nem_mx_vc_init,
    MPID_nem_mx_vc_destroy,
18
19
    MPID_nem_mx_vc_terminate,
    MPID_nem_mx_anysource_iprobe
20
21
};

22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
static MPIDI_Comm_ops_t comm_ops = {
    MPID_nem_mx_directRecv, /* recv_posted */
    
    MPID_nem_mx_directSend, /* send */
    MPID_nem_mx_directSend, /* rsend */
    MPID_nem_mx_directSsend, /* ssend */
    MPID_nem_mx_directSend, /* isend */
    MPID_nem_mx_directSend, /* irsend */
    MPID_nem_mx_directSsend, /* issend */
    
    NULL,                   /* send_init */
    NULL,                   /* bsend_init */
    NULL,                   /* rsend_init */
    NULL,                   /* ssend_init */
    NULL,                   /* startall */
    
    MPID_nem_mx_cancel_send,/* cancel_send */
39
40
41
42
    MPID_nem_mx_cancel_recv, /* cancel_recv */
    
    MPID_nem_mx_probe, /* probe */
    MPID_nem_mx_iprobe /* iprobe */
43
44
};

45

46
47
48
#define MPIDI_CH3I_ENDPOINT_KEY "endpoint_id"
#define MPIDI_CH3I_NIC_KEY      "nic_id"

49
50
51
52
53
int           MPID_nem_mx_pending_send_req = 0;
uint32_t      MPID_NEM_MX_FILTER = 0xabadbada;
uint64_t      MPID_nem_mx_local_nic_id;
uint32_t      MPID_nem_mx_local_endpoint_id;
mx_endpoint_t MPID_nem_mx_local_endpoint;
54
55
56
57
58

#undef FUNCNAME
#define FUNCNAME init_mx
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
59
static int init_mx( MPIDI_PG_t *pg_p )
60
{
61
62
63
64
65
66
   mx_endpoint_addr_t local_endpoint_addr;
   mx_return_t        ret;
   mx_param_t         param;
   int                mpi_errno = MPI_SUCCESS;
   int                r;

67
   r = MPL_putenv("MX_DISABLE_SHARED=1");
68
   MPIU_ERR_CHKANDJUMP(r, mpi_errno, MPI_ERR_OTHER, "**putenv");
69
   r = MPL_putenv("MX_DISABLE_SELF=1");
70
   MPIU_ERR_CHKANDJUMP(r, mpi_errno, MPI_ERR_OTHER, "**putenv");
71
72
73
74

   ret = mx_init();
   MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_init", "**mx_init %s", mx_strerror (ret));
   
75
76
   mx_set_error_handler(MX_ERRORS_RETURN);

77
78
79
80
81
82
83
84
85
86
87
88
89
90
   /*
   ret = mx_get_info(NULL, MX_NIC_COUNT, NULL, 0, &nic_count, sizeof(int));
   MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_get_info", "**mx_get_info %s", mx_strerror (ret));
   
   count = ++nic_count;
   mx_nics = (uint64_t *)MPIU_Malloc(count*sizeof(uint64_t));
   ret = mx_get_info(NULL, MX_NIC_IDS, NULL, 0, mx_nics, count*sizeof(uint64_t));
   MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_get_info", "**mx_get_info %s", mx_strerror (ret));
    
    do{	     
      ret = mx_nic_id_to_board_number(mx_nics[index],&mx_board_num);
      index++;
   }while(ret != MX_SUCCESS);
   */
91
92
93
94
95
96
97
98
#ifndef USE_CTXT_AS_MARK
   param.key = MX_PARAM_CONTEXT_ID;
   param.val.context_id.bits  = NEM_MX_MATCHING_BITS - SHIFT_TYPE;
   param.val.context_id.shift = SHIFT_TYPE;
   ret = mx_open_endpoint(MX_ANY_NIC,MX_ANY_ENDPOINT,MPID_NEM_MX_FILTER,&param,1,&MPID_nem_mx_local_endpoint);
#else
   ret = mx_open_endpoint(MX_ANY_NIC,MX_ANY_ENDPOINT,MPID_NEM_MX_FILTER,NULL,0,&MPID_nem_mx_local_endpoint);
#endif
99
100
   MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_open_endpoint", "**mx_open_endpoint %s", mx_strerror (ret));
      
101
   ret = mx_get_endpoint_addr(MPID_nem_mx_local_endpoint,&local_endpoint_addr);
102
103
   MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_get_endpoint_addr", "**mx_get_endpoint_addr %s", mx_strerror (ret));   
   
104
   ret = mx_decompose_endpoint_addr(local_endpoint_addr,&MPID_nem_mx_local_nic_id,&MPID_nem_mx_local_endpoint_id);
105
106
   MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_decompose_endpoint_addr", "**mx_decompose_endpoint_addr %s", mx_strerror (ret));
   
107
108
109
110
 fn_exit:
   return mpi_errno;
 fn_fail:
   goto fn_exit;
111
112
113
114
}

/*
 int  
115
   MPID_nem_mx_init(MPID_nem_queue_ptr_t proc_recv_queue, MPID_nem_queue_ptr_t proc_free_queue, MPID_nem_cell_ptr_t proc_elements, int num_proc_elements,
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
	          MPID_nem_cell_ptr_t module_elements, int num_module_elements, 
		  MPID_nem_queue_ptr_t *module_free_queue)

   IN
       proc_recv_queue -- main recv queue for the process
       proc_free_queue -- main free queueu for the process
       proc_elements -- pointer to the process' queue elements
       num_proc_elements -- number of process' queue elements
       module_elements -- pointer to queue elements to be used by this module
       num_module_elements -- number of queue elements for this module
   OUT
       free_queue -- pointer to the free queue for this module.  The process will return elements to
                     this queue
*/

#undef FUNCNAME
132
#define FUNCNAME MPID_nem_mx_init
133
134
135
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int
Darius Buntinas's avatar
Darius Buntinas committed
136
MPID_nem_mx_init (MPIDI_PG_t *pg_p, int pg_rank,
137
		  char **bc_val_p, int *val_max_sz_p)
138
139
{   
   int mpi_errno = MPI_SUCCESS ;
Guillaume Mercier's avatar
Guillaume Mercier committed
140
141

   MPID_nem_mx_internal_req_queue_init();
142
   
143
144
   mpi_errno = init_mx(pg_p);
   if (mpi_errno) MPIU_ERR_POP(mpi_errno);
145
146
147
148
149
150
151
152
153

   mpi_errno = MPID_nem_mx_get_business_card (pg_rank, bc_val_p, val_max_sz_p);
   if (mpi_errno) MPIU_ERR_POP (mpi_errno);

   mx_register_unexp_handler(MPID_nem_mx_local_endpoint,MPID_nem_mx_get_adi_msg,NULL);

   mpi_errno = MPIDI_CH3I_Register_anysource_notification(MPID_nem_mx_anysource_posted, 
							  MPID_nem_mx_anysource_matched);
   if (mpi_errno) MPIU_ERR_POP(mpi_errno);
154
155
156
157
158
159
160
161

   fn_exit:
       return mpi_errno;
   fn_fail:
       goto fn_exit;
}

#undef FUNCNAME
162
#define FUNCNAME MPID_nem_mx_get_business_card
163
164
165
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int
166
MPID_nem_mx_get_business_card (int my_rank, char **bc_val_p, int *val_max_sz_p)
167
{
168
169
170
171
172
173
174
175
176
177
178
179
180
181
    int mpi_errno = MPI_SUCCESS;
    int str_errno = MPIU_STR_SUCCESS;

    str_errno = MPIU_Str_add_int_arg (bc_val_p, val_max_sz_p, MPIDI_CH3I_ENDPOINT_KEY, MPID_nem_mx_local_endpoint_id);
    if (str_errno) {
        MPIU_ERR_CHKANDJUMP(str_errno == MPIU_STR_NOMEM, mpi_errno, MPI_ERR_OTHER, "**buscard_len");
        MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**buscard");
    }
    
    str_errno = MPIU_Str_add_binary_arg (bc_val_p, val_max_sz_p, MPIDI_CH3I_NIC_KEY, (char *)&MPID_nem_mx_local_nic_id, sizeof(uint64_t));
    if (str_errno) {
        MPIU_ERR_CHKANDJUMP(str_errno == MPIU_STR_NOMEM, mpi_errno, MPI_ERR_OTHER, "**buscard_len");
        MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**buscard");
    }
182
   
183
184
185
186
 fn_exit:
     return mpi_errno;
 fn_fail:
     goto fn_exit;
187
188
189
}

#undef FUNCNAME
190
#define FUNCNAME MPID_nem_mx_get_from_bc
191
192
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
193
int MPID_nem_mx_get_from_bc(const char *business_card, uint32_t *remote_endpoint_id, uint64_t *remote_nic_id)
194
195
{
   int mpi_errno = MPI_SUCCESS;
196
   int str_errno = MPIU_STR_SUCCESS;
197
   int len;
198
   int tmp_endpoint_id;
199
   
200
   mpi_errno = MPIU_Str_get_int_arg(business_card, MPIDI_CH3I_ENDPOINT_KEY, &tmp_endpoint_id);
201
202
   /* FIXME: create a real error string for this */
   MPIU_ERR_CHKANDJUMP(str_errno, mpi_errno, MPI_ERR_OTHER, "**argstr_hostd");
203
204
205
   *remote_endpoint_id = (uint32_t)tmp_endpoint_id;
   
   mpi_errno = MPIU_Str_get_binary_arg (business_card, MPIDI_CH3I_NIC_KEY, (char *)remote_nic_id, sizeof(uint64_t), &len);
206
207
   /* FIXME: create a real error string for this */
   MPIU_ERR_CHKANDJUMP(str_errno || len != sizeof(uint64_t), mpi_errno, MPI_ERR_OTHER, "**argstr_hostd");
208
209
210
   
   fn_exit:
     return mpi_errno;
211
   fn_fail:
212
213
214
215
     goto fn_exit;
}

#undef FUNCNAME
216
#define FUNCNAME MPID_nem_mx_connect_to_root
217
218
219
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int
220
MPID_nem_mx_connect_to_root (const char *business_card, MPIDI_VC_t *new_vc)
221
{
222
   return MPI_SUCCESS;
223
224
225
}

#undef FUNCNAME
226
#define FUNCNAME MPID_nem_mx_vc_init
227
228
229
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int
230
MPID_nem_mx_vc_init (MPIDI_VC_t *vc)
231
{
232
233
   uint32_t threshold;
   MPIDI_CH3I_VC *vc_ch = (MPIDI_CH3I_VC *)vc->channel_private;
234
   int mpi_errno = MPI_SUCCESS;
235

236
   /* first make sure that our private fields in the vc fit into the area provided  */
237
   MPIU_Assert(sizeof(MPID_nem_mx_vc_area) <= MPID_NEM_VC_NETMOD_AREA_LEN);
238

239
240
241
242
243
#ifdef ONDEMAND
   VC_FIELD(vc, local_connected)  = 0;
   VC_FIELD(vc, remote_connected) = 0;
#else
   {
244
245
246
247
248
249
250
251
252
253
       char *business_card;
       int   val_max_sz;
       int   ret;
#ifdef USE_PMI2_API
       val_max_sz = PMI2_MAX_VALLEN;
#else
       mpi_errno = PMI_KVS_Get_value_length_max(&val_max_sz);
#endif 
       business_card = (char *)MPIU_Malloc(val_max_sz); 
       mpi_errno = vc->pg->getConnInfo(vc->pg_rank, business_card,val_max_sz, vc->pg);
254
255
256
257
       if (mpi_errno) MPIU_ERR_POP(mpi_errno);
       
       mpi_errno = MPID_nem_mx_get_from_bc (business_card, &VC_FIELD(vc, remote_endpoint_id), &VC_FIELD(vc, remote_nic_id));
       if (mpi_errno)    MPIU_ERR_POP (mpi_errno);
258
259
260

       MPIU_Free(business_card);
       
261
262
263
264
       ret = mx_connect(MPID_nem_mx_local_endpoint,VC_FIELD(vc, remote_nic_id),VC_FIELD(vc, remote_endpoint_id),
			MPID_NEM_MX_FILTER,MX_INFINITE,&(VC_FIELD(vc, remote_endpoint_addr)));
       MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_connect", "**mx_connect %s", mx_strerror (ret));
       mx_set_endpoint_addr_context(VC_FIELD(vc, remote_endpoint_addr),(void *)vc);
265
266

       MPIDI_CHANGE_VC_STATE(vc, ACTIVE);
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
   }
#endif
   mx_get_info(MPID_nem_mx_local_endpoint, MX_COPY_SEND_MAX, NULL, 0, &threshold, sizeof(uint32_t));

   vc->eager_max_msg_sz = threshold;
   vc->rndvSend_fn      = NULL;
   vc->sendNoncontig_fn = MPID_nem_mx_SendNoncontig;
   vc->comm_ops         = &comm_ops;
 
   vc_ch->iStartContigMsg = MPID_nem_mx_iStartContigMsg;
   vc_ch->iSendContig     = MPID_nem_mx_iSendContig;

 fn_exit:
   return mpi_errno;
 fn_fail:
   goto fn_exit;
283
284
285
}

#undef FUNCNAME
286
#define FUNCNAME MPID_nem_mx_vc_destroy
287
288
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
289
int MPID_nem_mx_vc_destroy(MPIDI_VC_t *vc)
290
291
292
293
294
{
    int mpi_errno = MPI_SUCCESS;   

    /* free any resources associated with this VC here */

295
    return mpi_errno;
296
297
298
}

#undef FUNCNAME
299
#define FUNCNAME MPID_nem_mx_vc_terminate
300
301
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
302
int MPID_nem_mx_vc_terminate (MPIDI_VC_t *vc)
303
304
305
{
    return MPI_SUCCESS;
}