Commit 528f01fc authored by David Goodell's avatar David Goodell
Browse files

[svn-r3629] Cleanup the node topology routines so that they integrate better with non-CH3 devices.

Reviewed by buntinas@.
parent cf2c6e2a
......@@ -1468,7 +1468,13 @@ void MPIR_CommL_forget( MPID_Comm * );
/* ------------------------------------------------------------------------- */
/* Prototypes and definitions for the node ID code. This is used to support
hierarchical collectives in a (mostly) device-independent way. */
#if defined(MPID_USE_NODE_IDS)
/* MPID_Node_id_t is a signed integer type defined by the device in mpidpre.h. */
int MPID_Get_node_id(MPID_Comm *comm, int rank, MPID_Node_id_t *id_p);
int MPID_Get_max_node_id(MPID_Comm *comm, MPID_Node_id_t *max_id_p);
#endif
/* ------------------------------------------------------------------------- */
/*S
......@@ -3684,12 +3690,10 @@ int vsnprintf(char *str, size_t size, const char *format, va_list ap);
/* Routines for determining local and remote processes */
int MPIU_Get_local_procs(int global_rank, int num_global, int *num_local_p, int **local_procs_p, int *local_rank_p);
int MPIU_Find_local_and_external(struct MPID_Comm *comm, int *local_size_p, int *local_rank_p, int **local_ranks_p,
int *external_size_p, int *external_rank_p, int **external_ranks_p,
int **intranode_table, int **internode_table_p);
int MPIU_Get_internode_rank(MPID_Comm *comm_ptr, int r);
int MPIU_Get_intranode_rank(MPID_Comm *comm_ptr, int r);
int MPIU_Local_procs_finalize(void);
#endif /* MPIIMPL_INCLUDED */
......@@ -36,6 +36,8 @@ int MPID_nem_finalize()
MPIU_Free(MPID_nem_mem_region.mailboxes.out);
MPIU_Free(MPID_nem_mem_region.mailboxes.in);
MPIU_Free(MPID_nem_mem_region.local_procs);
#ifdef MEM_REGION_IN_HEAP
MPIU_Free(MPID_nem_mem_region_ptr);
#endif /* MEM_REGION_IN_HEAP */
......
......@@ -24,6 +24,9 @@ char MPID_nem_hostname[MAX_HOSTNAME_LEN] = "UNKNOWN";
static MPID_nem_queue_ptr_t net_free_queue;
static int get_local_procs(MPIDI_PG_t *pg, int our_pg_rank, int *num_local_p,
int **local_procs_p, int *local_rank_p);
#ifndef MIN
#define MIN( a , b ) ((a) > (b)) ? (b) : (a)
#endif /* MIN */
......@@ -101,7 +104,7 @@ _MPID_nem_init (int pg_rank, MPIDI_PG_t *pg_p, int ckpt_restart, int has_parent)
MPID_nem_hostname[MAX_HOSTNAME_LEN-1] = '\0';
mpi_errno = MPIU_Get_local_procs(pg_rank, num_procs, &num_local, &local_procs, &local_rank);
mpi_errno = get_local_procs(pg_p, pg_rank, &num_local, &local_procs, &local_rank);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
#ifdef MEM_REGION_IN_HEAP
......@@ -551,3 +554,59 @@ int MPID_nem_connect_to_root (const char *business_card, MPIDI_VC_t *new_vc)
{
return MPID_nem_netmod_func->connect_to_root (business_card, new_vc);
}
/* get_local_procs() determines which processes are local and
should use shared memory
If an output variable pointer is NULL, it won't be set.
Caller should NOT free any returned buffers.
Note that this is really only a temporary solution as it only
calculates these values for processes MPI_COMM_WORLD, i.e., not for
spawned or attached processes.
*/
#undef FUNCNAME
#define FUNCNAME get_local_procs
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
static int get_local_procs(MPIDI_PG_t *pg, int our_pg_rank, int *num_local_p,
int **local_procs_p, int *local_rank_p)
{
int mpi_errno = MPI_SUCCESS;
int *procs;
int i;
int num_local = 0;
MPID_Node_id_t our_node_id;
MPIU_CHKPMEM_DECL(1);
MPIU_Assert(our_pg_rank < pg->size);
our_node_id = pg->vct[our_pg_rank].node_id;
MPIU_CHKPMEM_MALLOC(procs, int *, pg->size * sizeof(int), mpi_errno, "local process index array");
for (i = 0; i < pg->size; ++i) {
if (our_node_id == pg->vct[i].node_id) {
if (i == our_pg_rank && local_rank_p != NULL) {
*local_rank_p = num_local;
}
procs[num_local] = i;
++num_local;
}
}
MPIU_CHKPMEM_COMMIT();
if (num_local_p != NULL)
*num_local_p = num_local;
if (local_procs_p != NULL)
*local_procs_p = procs;
fn_exit:
return mpi_errno;
fn_fail:
/* --BEGIN ERROR HANDLING-- */
MPIU_CHKPMEM_REAP();
goto fn_exit;
/* --END ERROR HANDLING-- */
}
......@@ -651,6 +651,9 @@ typedef struct MPIDI_VC
/* Local process ID */
int lpid;
/* The node id of this process, used for topologically aware collectives. */
MPID_Node_id_t node_id;
/* port name tag */
int port_name_tag; /* added to handle dynamic process mgmt */
......@@ -1169,6 +1172,9 @@ int MPIDI_CH3I_BCFree( char *publish_bc );
card) */
int MPIDI_PG_SetConnInfo( int rank, const char *connString );
/* Fill in the node_id information for each VC in the given PG. */
int MPIDI_Populate_vc_node_ids(MPIDI_PG_t *pg, int our_pg_rank);
/* NOTE: Channel function prototypes are in mpidi_ch3_post.h since some of the
macros require their declarations. */
......
......@@ -53,6 +53,12 @@ typedef int16_t MPIR_Rank_t;
typedef int32_t MPIR_Rank_t;
#endif /* CH3_RANK_BITS */
/* Indicates that this device is topology aware and implements the
MPID_Get_node_id function (and friends). */
#define MPID_USE_NODE_IDS
typedef MPIR_Rank_t MPID_Node_id_t;
/* For the typical communication system for which the ch3 channel is
appropriate, 16 bits is sufficient for the rank. By also using 16
bits for the context, we can reduce the size of the match
......
......@@ -179,6 +179,8 @@ void MPIDI_CH3_Request_destroy(MPID_Request * req)
* by the segment contained in the request structure.
* If the density of IOV is not sufficient, pack the data into a send/receive
* buffer and point the IOV at the buffer.
*
* Expects sreq->dev.OnFinal to be initialized (even if it's NULL).
*/
#undef FUNCNAME
#define FUNCNAME MPIDI_CH3U_Request_load_send_iov
......
......@@ -143,9 +143,6 @@ int MPID_Finalize(void)
}
}
mpi_errno = MPIU_Local_procs_finalize();
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_FINALIZE);
return mpi_errno;
......
......@@ -55,8 +55,8 @@ int MPID_Init(int *argc, char ***argv, int requested, int *provided,
/* This is a sanity check because we define a generic packet size
*/
if (sizeof(MPIDI_CH3_PktGeneric_t) < sizeof(MPIDI_CH3_Pkt_t)) {
fprintf( stderr, "Internal error - packet definition is too small. Generic is %d bytes, MPIDI_CH3_Pkt_t is %d\n", sizeof(MPIDI_CH3_PktGeneric_t),
sizeof(MPIDI_CH3_Pkt_t) );
fprintf( stderr, "Internal error - packet definition is too small. Generic is %ld bytes, MPIDI_CH3_Pkt_t is %ld\n", (long int)sizeof(MPIDI_CH3_PktGeneric_t),
(long int)sizeof(MPIDI_CH3_Pkt_t) );
exit(1);
}
#endif
......@@ -93,6 +93,12 @@ int MPID_Init(int *argc, char ***argv, int requested, int *provided,
/* FIXME: Why do we add a ref to pg here? */
MPIDI_PG_add_ref(pg);
/* We intentionally call this before the channel init so that the channel
can use the node_id info. */
/* Ideally this wouldn't be needed. Once we have PMIv2 support for node
information we should probably eliminate this function. */
mpi_errno = MPIDI_Populate_vc_node_ids(pg, pg_rank);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
/*
* Let the channel perform any necessary initialization
......@@ -105,9 +111,6 @@ int MPID_Init(int *argc, char ***argv, int requested, int *provided,
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER, "**ch3|ch3_init");
}
mpi_errno = MPIU_Get_local_procs(pg_rank, pg_size, NULL, NULL, NULL);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
/*
* Initialize the MPI_COMM_WORLD object
*/
......@@ -422,23 +425,8 @@ static int InitPG( int *argc, char ***argv,
connection information by passing the pg to the channel init routine */
if (usePMI) {
/* Tell the process group how to get connection information */
MPIDI_PG_InitConnKVS( pg );
#if 0
/* If we're supporting the debugger, we can save our host and
pid here. This publishes the data in the kvs space.
This allows the MPI processes to access the information
about the other processes without any PMI changes. */
#ifdef HAVE_DEBUGGER_SUPPORT
{
char key[64];
char myinfo[512];
MPIU_Snprintf( key, sizeof(key), "hpid-%d", pg_rank );
MPIU_Snpritnf( myinfo, sizeof(key), "%s:%d", hostname, getpid() );
PMI_KVS_Put( pg->world->connData, key, myinfo );
}
#endif
#endif
mpi_errno = MPIDI_PG_InitConnKVS( pg );
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
/* FIXME: Who is this for and where does it belong? */
......
......@@ -678,6 +678,7 @@ int MPIDI_VC_Init( MPIDI_VC_t *vc, MPIDI_PG_t *pg, int rank )
vc->pg = pg;
vc->pg_rank = rank;
vc->lpid = lpid_counter++;
vc->node_id = -1;
MPIDI_VC_Init_seqnum_send(vc);
MPIDI_VC_Init_seqnum_recv(vc);
vc->rndvSend_fn = MPIDI_CH3_RndvSend;
......@@ -689,3 +690,221 @@ int MPIDI_VC_Init( MPIDI_VC_t *vc, MPIDI_PG_t *pg, int rank )
return MPI_SUCCESS;
}
/* ----------------------------------------------------------------------- */
/* Routines to vend topology information. */
static MPID_Node_id_t g_num_nodes = 0;
char MPIU_hostname[MAX_HOSTNAME_LEN] = "_UNKNOWN_"; /* '_' is an illegal char for a hostname so */
/* this will never match */
#undef FUNCNAME
#define FUNCNAME MPID_Get_node_id
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPID_Get_node_id(MPID_Comm *comm, int rank, MPID_Node_id_t *id_p)
{
*id_p = comm->vcr[rank]->node_id;
return MPI_SUCCESS;
}
#undef FUNCNAME
#define FUNCNAME MPID_Get_max_node_id
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
/* Providing a comm argument permits optimization, but this function is always
allowed to return the max for the universe. */
int MPID_Get_max_node_id(MPID_Comm *comm, MPID_Node_id_t *max_id_p)
{
/* easiest way to implement this is to track it at PG create/destroy time */
*max_id_p = g_num_nodes - 1;
MPIU_Assert(*max_id_p >= 0);
return MPI_SUCCESS;
}
static int publish_node_id(MPIDI_PG_t *pg, int our_pg_rank)
{
int mpi_errno = MPI_SUCCESS;
int pmi_errno;
int ret;
char *key;
int key_max_sz;
char *kvs_name;
MPIU_CHKLMEM_DECL(1);
/* set MPIU_hostname */
ret = gethostname(MPIU_hostname, MAX_HOSTNAME_LEN);
MPIU_ERR_CHKANDJUMP2(ret == -1, mpi_errno, MPI_ERR_OTHER, "**sock_gethost", "**sock_gethost %s %d", strerror(errno), errno);
MPIU_hostname[MAX_HOSTNAME_LEN-1] = '\0';
/* Allocate space for pmi key */
pmi_errno = PMI_KVS_Get_key_length_max(&key_max_sz);
MPIU_ERR_CHKANDJUMP1(pmi_errno, mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %d", pmi_errno);
MPIU_CHKLMEM_MALLOC(key, char *, key_max_sz, mpi_errno, "key");
mpi_errno = MPIDI_PG_GetConnKVSname(&kvs_name);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
/* Put my hostname id */
if (pg->size > 1)
{
memset(key, 0, key_max_sz);
MPIU_Snprintf(key, key_max_sz, "hostname[%d]", our_pg_rank);
pmi_errno = PMI_KVS_Put(kvs_name, key, MPIU_hostname);
MPIU_ERR_CHKANDJUMP1(pmi_errno != PMI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**pmi_kvs_put", "**pmi_kvs_put %d", pmi_errno);
pmi_errno = PMI_KVS_Commit(kvs_name);
MPIU_ERR_CHKANDJUMP1(pmi_errno != PMI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**pmi_kvs_commit", "**pmi_kvs_commit %d", pmi_errno);
pmi_errno = PMI_Barrier();
MPIU_ERR_CHKANDJUMP1(pmi_errno != PMI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**pmi_barrier", "**pmi_barrier %d", pmi_errno);
}
fn_exit:
MPIU_CHKLMEM_FREEALL();
return mpi_errno;
fn_fail:
goto fn_exit;
}
/* Fills in the node_id info from PMI info. Adapted from MPIU_Get_local_procs.
This function is collective over the entire PG because PMI_Barrier is called.
our_pg_rank should be set to -1 if this is not the current process' PG. This
is currently not supported due to PMI limitations.
Algorithm:
Each process kvs_puts its hostname and stores the total number of
processes (g_num_global). Each process determines the number of nodes
(g_num_nodes) and assigns a node id to each process (g_node_ids[]):
For each hostname the process seaches the list of unique nodes
names (node_names[]) for a match. If a match is found, the node id
is recorded for that matching process. Otherwise, the hostname is
added to the list of node names.
*/
int MPIDI_Populate_vc_node_ids(MPIDI_PG_t *pg, int our_pg_rank)
{
int mpi_errno = MPI_SUCCESS;
int pmi_errno;
int ret;
int val;
int i, j;
char *key;
int key_max_sz;
char *kvs_name;
char **node_names;
char *node_name_buf;
int no_local = 0;
int odd_even_cliques = 0;
MPIU_CHKLMEM_DECL(3);
if (our_pg_rank == -1) {
/* FIXME this routine can't handle the dynamic process case at this
time. This will require more support from the process manager. */
MPIU_Assert(0);
}
mpi_errno = publish_node_id(pg, our_pg_rank);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
/* Used for debugging only. This disables communication over shared memory */
#ifdef ENABLED_NO_LOCAL
no_local = 1;
#else
ret = MPIU_GetEnvBool("MPICH_NO_LOCAL", &val);
if (ret == 1 && val)
no_local = 1;
#endif
/* Used for debugging on a single machine: Odd procs on a node are
seen as local to each other, and even procs on a node are seen
as local to each other. */
#ifdef ENABLED_ODD_EVEN_CLIQUES
odd_even_cliques = 1;
#else
ret = MPIU_GetEnvBool("MPICH_ODD_EVEN_CLIQUES", &val);
if (ret == 1 && val)
odd_even_cliques = 1;
#endif
if (no_local) {
/* just assign 0 to n-1 as node ids and bail */
for (i = 0; i < pg->size; ++i) {
pg->vct[i].node_id = g_num_nodes++;
}
goto fn_exit;
}
/* Allocate space for pmi key */
pmi_errno = PMI_KVS_Get_key_length_max(&key_max_sz);
MPIU_ERR_CHKANDJUMP1(pmi_errno, mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %d", pmi_errno);
MPIU_CHKLMEM_MALLOC(key, char *, key_max_sz, mpi_errno, "key");
mpi_errno = MPIDI_PG_GetConnKVSname(&kvs_name);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
/* Allocate temporary structures. These would need to be persistent if
we somehow were able to support dynamic processes via this method. */
MPIU_CHKLMEM_MALLOC(node_names, char **, pg->size * sizeof(char*), mpi_errno, "node_names");
MPIU_CHKLMEM_MALLOC(node_name_buf, char *, pg->size * key_max_sz * sizeof(char), mpi_errno, "node_name_buf");
/* Gather hostnames */
for (i = 0; i < pg->size; ++i)
{
node_names[i] = &node_name_buf[i * key_max_sz];
node_names[i][0] = '\0';
}
for (i = 0; i < pg->size; ++i)
{
if (i == our_pg_rank)
{
/* This is us, no need to perform a get */
MPIU_Snprintf(node_names[g_num_nodes], key_max_sz, "%s", MPIU_hostname);
}
else
{
memset(key, 0, key_max_sz);
MPIU_Snprintf(key, key_max_sz, "hostname[%d]", i);
pmi_errno = PMI_KVS_Get(kvs_name, key, node_names[g_num_nodes], key_max_sz);
MPIU_ERR_CHKANDJUMP1(pmi_errno != PMI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**pmi_kvs_get", "**pmi_kvs_get %d", pmi_errno);
}
/* Find the node_id for this process, or create a new one */
/* FIXME:need a better algorithm -- this one does O(N^2) strncmp()s! */
/* The right fix is to get all this information from the process
manager, rather than bother with this hostname hack at all. */
for (j = 0; j < g_num_nodes; ++j)
if (!strncmp(node_names[j], node_names[g_num_nodes], key_max_sz))
break;
if (j == g_num_nodes)
++g_num_nodes;
else
node_names[g_num_nodes][0] = '\0';
pg->vct[i].node_id = j;
}
if (odd_even_cliques)
{
/* Create new processes for all odd numbered processes. This
may leave nodes ids with no processes assigned to them, but
I think this is OK */
for (i = 0; i < pg->size; ++i)
if (i & 0x1)
pg->vct[i].node_id += g_num_nodes;
g_num_nodes *= 2;
}
fn_exit:
MPIU_CHKLMEM_FREEALL();
return mpi_errno;
fn_fail:
goto fn_exit;
}
......@@ -18,278 +18,6 @@
#include <errno.h>
#endif
char MPIU_hostname[MAX_HOSTNAME_LEN] = "_UNKNOWN_"; /* '_' is an illegal char for a hostname so this will never match */
static int local_procs_initialized = 0; /* flag indicating whether the below fields are valid */
static int g_num_global; /* number of processes in comm world */
static int g_num_local; /* number of local processes */
static int *g_local_procs; /* array of global ranks of local processes */
static int g_local_rank; /* local rank of this process */
static int g_num_nodes; /* number of local nodes */
static int *g_node_ids; /* g_node_ids[rank] gives the id of the node where rank is running */
static int get_local_procs_nolocal(int global_rank, int num_global);
/* MPIU_Get_local_procs() determines which processes are local and
should use shared memory
This uses PMI to get all of the processes that have the same
hostname, and puts them into local_procs sorted by global rank.
If an output variable pointer is NULL, it won't be set.
Caller should NOT free any returned buffers.
Note that this is really only a temporary solution as it only
calculates these values for processes MPI_COMM_WORLD, i.e., not for
spawned or attached processes.
Algorithm:
Each process kvs_puts its hostname and stores the total number of
processes (g_num_global).
Each process determines local rank (g_local_rank), the number of
processes local to this node (g_num_local) and creates a list of the
local processes (g_local_procs[]):
The process kvs_gets the hostname of every other process, and
compares each to its own hostname. When it finds a match, we
assume that the matched process is on the same node as this one, so
the process records the global rank of the matching process in the
procs[] array. If the matching process turns out to be the process
itself, it now knows its local rank, and sets it appropriately.
Note: There is an additional clause for matching hostnames when
"odd_even_cliques" is enabled which checks that the local process
and the matched process are either both even or both odd.
Each process determines the number of nodes (g_num_nodes) and assigns
a node id to each process (g_node_ids[]):
For each hostname the process seaches the list of unique nodes
names (node_names[]) for a match. If a match is found, the node id
is recorded for that matching process. Otherwise, the hostname is
added to the list of node names.
*/
#undef FUNCNAME
#define FUNCNAME MPIU_Get_local_procs
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPIU_Get_local_procs(int global_rank, int num_global, int *num_local_p, int **local_procs_p, int *local_rank_p)
{
int mpi_errno = MPI_SUCCESS;
int pmi_errno;
int ret;
int val;
int *procs;
int i, j;
char *key;
int key_max_sz;
char *kvs_name;
char **node_names;
char *node_name_buf;
int no_local = 0;
int odd_even_cliques = 0;
MPIU_CHKPMEM_DECL(2);
MPIU_CHKLMEM_DECL(3);
if (local_procs_initialized)
goto fn_return_values;
/* Used for debugging only. This disables communication over shared memory */
#ifdef ENABLED_NO_LOCAL
no_local = 1;
#else
ret = MPIU_GetEnvBool("MPICH_NO_LOCAL", &val);
if (ret == 1 && val)
no_local = 1;
#endif
/* Used for debugging on a single machine: Odd procs on a node are
seen as local to each other, and even procs on a node are seen
as local to each other. */
#ifdef ENABLED_ODD_EVEN_CLIQUES
odd_even_cliques = 1;
#else
ret = MPIU_GetEnvBool("MPICH_ODD_EVEN_CLIQUES", &val);
if (ret == 1 && val)
odd_even_cliques = 1;
#endif
/* set MPIU_hostname */
ret = gethostname(MPIU_hostname, MAX_HOSTNAME_LEN);
MPIU_ERR_CHKANDJUMP2(ret == -1, mpi_errno, MPI_ERR_OTHER, "**sock_gethost", "**sock_gethost %s %d", strerror(errno), errno);
MPIU_hostname[MAX_HOSTNAME_LEN-1] = '\0';
if (no_local)
{
mpi_errno = get_local_procs_nolocal(global_rank, num_global);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
goto fn_return_values;
}
g_num_global = num_global;
/* Allocate space for pmi key */
pmi_errno = PMI_KVS_Get_key_length_max(&key_max_sz);
MPIU_ERR_CHKANDJUMP1(pmi_errno, mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %d", pmi_errno);
MPIU_CHKLMEM_MALLOC(key, char *, key_max_sz, mpi_errno, "key");
mpi_errno = MPIDI_PG_GetConnKVSname(&kvs_name);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
/* Put my hostname id */
if (num_global > 1)
{
memset(key, 0, key_max_sz);
MPIU_Snprintf(key, key_max_sz, "hostname[%d]", global_rank);
pmi_errno = PMI_KVS_Put(kvs_name, key, MPIU_hostname);
MPIU_ERR_CHKANDJUMP1(pmi_errno != PMI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**pmi_kvs_put", "**pmi_kvs_put %d", pmi_errno);
pmi_errno = PMI_KVS_Commit(kvs_name);
MPIU_ERR_CHKANDJUMP1(pmi_errno != PMI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**pmi_kvs_commit", "**pmi_kvs_commit %d", pmi_errno);
pmi_errno = PMI_Barrier();
MPIU_ERR_CHKANDJUMP1(pmi_errno != PMI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**pmi_barrier", "**pmi_barrier %d", pmi_errno);
}
/* allocate structures */
MPIU_CHKPMEM_MALLOC(procs, int *, num_global * sizeof(int), mpi_errno, "local process index array");
MPIU_CHKPMEM_MALLOC(g_node_ids, int *, num_global * sizeof(int), mpi_errno, "node_ids");
MPIU_CHKLMEM_MALLOC(node_names, char **, num_global * sizeof(char*), mpi_errno, "node_names");
MPIU_CHKLMEM_MALLOC(node_name_buf, char *, num_global * key_max_sz * sizeof(char), mpi_errno, "node_name_buf");
/* Gather hostnames */
for (i = 0; i < num_global; ++i)
{