Commit c13ab18f authored by Pavan Balaji's avatar Pavan Balaji Committed by Kenneth Raffenetti
Browse files

Move channel specific code into a separate channel structure.



Also move device specific comm structure components to 'dev' to clean
up the naming a bit.
Signed-off-by: Kenneth Raffenetti's avatarKen Raffenetti <raffenet@mcs.anl.gov>
parent bbacb5b5
......@@ -28,6 +28,10 @@
MPIDI_NEM_PKT_LMT_COOKIE, \
MPIDI_NEM_PKT_CKPT_MARKER, \
MPIDI_NEM_PKT_NETMOD
typedef struct {
struct MPID_nem_barrier_vars *barrier_vars; /* shared memory variables used in barrier */
} MPIDI_CH3I_CH_comm_t;
typedef enum MPIDI_CH3I_VC_state
{
......
......@@ -35,7 +35,7 @@ int MPIDI_CH3I_comm_create(MPID_Comm *comm, void *param)
/* set up intranode barrier iff this is an intranode communicator */
if (comm->hierarchy_kind == MPID_HIERARCHY_NODE) {
MPID_Collops *cf, **cf_p;
comm->ch.barrier_vars = NULL;
comm->dev.ch.barrier_vars = NULL;
/* We can't use a static coll_fns override table for our collectives
because we store a pointer to the previous coll_fns in our coll_fns
......@@ -109,9 +109,9 @@ int MPIDI_CH3I_comm_destroy(MPID_Comm *comm, void *param)
MPIU_Free(cf);
}
if (comm->ch.barrier_vars && OPA_fetch_and_decr_int(&comm->ch.barrier_vars->usage_cnt) == 1) {
if (comm->dev.ch.barrier_vars && OPA_fetch_and_decr_int(&comm->dev.ch.barrier_vars->usage_cnt) == 1) {
OPA_write_barrier();
OPA_store_int(&comm->ch.barrier_vars->context_id, NULL_CONTEXT_ID);
OPA_store_int(&comm->dev.ch.barrier_vars->context_id, NULL_CONTEXT_ID);
}
}
......@@ -178,11 +178,11 @@ static int barrier(MPID_Comm *comm_ptr, int *errflag)
time */
MPIDU_ERR_CHECK_MULTIPLE_THREADS_ENTER (comm_ptr);
if (comm_ptr->ch.barrier_vars == NULL) {
mpi_errno = alloc_barrier_vars (comm_ptr, &comm_ptr->ch.barrier_vars);
if (comm_ptr->dev.ch.barrier_vars == NULL) {
mpi_errno = alloc_barrier_vars (comm_ptr, &comm_ptr->dev.ch.barrier_vars);
if (mpi_errno) MPIU_ERR_POP (mpi_errno);
if (comm_ptr->ch.barrier_vars == NULL) {
if (comm_ptr->dev.ch.barrier_vars == NULL) {
/* no barrier_vars left -- revert to default barrier. */
/* FIXME: need a better solution here. e.g., allocate
some barrier_vars on the first barrier for the life of
......@@ -201,7 +201,7 @@ static int barrier(MPID_Comm *comm_ptr, int *errflag)
}
}
barrier_vars = comm_ptr->ch.barrier_vars;
barrier_vars = comm_ptr->dev.ch.barrier_vars;
sense = OPA_load_int(&barrier_vars->sig);
OPA_read_barrier();
......
......@@ -28,6 +28,10 @@ MPIDI_CH3I_PKT_SC_CONN_ACCEPT, \
MPIDI_CH3I_PKT_SC_OPEN_RESP, \
MPIDI_CH3I_PKT_SC_CLOSE
typedef struct {
int dummy; /* dummy variable to ensure we don't have an empty structure */
} MPIDI_CH3I_CH_comm_t;
/* This channel has no special channel data for the process group structure */
/* FIXME: Explain these; why is this separate from the VC state? */
......
......@@ -1961,8 +1961,8 @@ int MPIDI_CH3_ReqHandler_GetSendRespComplete( MPIDI_VC_t *, MPID_Request *,
#define MPIDI_CH3_GET_EAGER_THRESHOLD(eager_threshold_p, comm, vc) \
do { \
if ((comm)->ch.eager_max_msg_sz != -1) \
*(eager_threshold_p) = (comm)->ch.eager_max_msg_sz; \
if ((comm)->dev.eager_max_msg_sz != -1) \
*(eager_threshold_p) = (comm)->dev.eager_max_msg_sz; \
else \
*(eager_threshold_p) = (vc)->eager_max_msg_sz; \
} while (0)
......
......@@ -166,7 +166,7 @@ typedef union {
#define MPID_Dev_comm_create_hook(comm_) MPIDI_CH3I_Comm_create_hook(comm_)
#define MPID_Dev_comm_destroy_hook(comm_) MPIDI_CH3I_Comm_destroy_hook(comm_)
#define MPIDI_CH3I_Comm_AS_enabled(comm) ((comm)->ch.anysource_enabled)
#define MPIDI_CH3I_Comm_AS_enabled(comm) ((comm)->dev.anysource_enabled)
typedef struct MPIDI_CH3I_comm
{
......@@ -176,13 +176,13 @@ typedef struct MPIDI_CH3I_comm
int waiting_for_revoke; /* The number of other processes from which we are
* waiting for a revoke message before we can release
* the context id */
struct MPID_nem_barrier_vars *barrier_vars; /* shared memory variables used in barrier */
struct MPID_Comm *next; /* next pointer for list of communicators */
struct MPID_Comm *prev; /* prev pointer for list of communicators */
MPIDI_CH3I_CH_comm_t ch;
}
MPIDI_CH3I_comm_t;
#define MPID_DEV_COMM_DECL MPIDI_CH3I_comm_t ch;
#define MPID_DEV_COMM_DECL MPIDI_CH3I_comm_t dev;
#ifndef HAVE_MPIDI_VCRT
#define HAVE_MPIDI_VCRT
......
......@@ -12,9 +12,9 @@ static int comm_created(MPID_Comm *comm, void *param);
static int comm_destroyed(MPID_Comm *comm, void *param);
/* macros and head for list of communicators */
#define COMM_ADD(comm) MPL_DL_PREPEND_NP(comm_list, comm, ch.next, ch.prev)
#define COMM_DEL(comm) MPL_DL_DELETE_NP(comm_list, comm, ch.next, ch.prev)
#define COMM_FOREACH(elt) MPL_DL_FOREACH_NP(comm_list, elt, ch.next, ch.prev)
#define COMM_ADD(comm) MPL_DL_PREPEND_NP(comm_list, comm, dev.next, dev.prev)
#define COMM_DEL(comm) MPL_DL_DELETE_NP(comm_list, comm, dev.next, dev.prev)
#define COMM_FOREACH(elt) MPL_DL_FOREACH_NP(comm_list, elt, dev.next, dev.prev)
static MPID_Comm *comm_list = NULL;
typedef struct hook_elt
......@@ -203,13 +203,13 @@ int comm_created(MPID_Comm *comm, void *param)
MPIDI_FUNC_ENTER(MPID_STATE_COMM_CREATED);
comm->ch.anysource_enabled = TRUE;
comm->dev.anysource_enabled = TRUE;
/* Use the VC's eager threshold by default. */
comm->ch.eager_max_msg_sz = -1;
comm->dev.eager_max_msg_sz = -1;
/* Initialize the last acked failure to -1 */
comm->ch.last_ack_rank = -1;
comm->dev.last_ack_rank = -1;
COMM_ADD(comm);
......@@ -232,8 +232,8 @@ int comm_destroyed(MPID_Comm *comm, void *param)
MPIDI_FUNC_ENTER(MPID_STATE_COMM_DESTROYED);
COMM_DEL(comm);
comm->ch.next = NULL;
comm->ch.prev = NULL;
comm->dev.next = NULL;
comm->dev.prev = NULL;
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_COMM_DESTROYED);
......@@ -307,7 +307,7 @@ int MPIDI_CH3I_Comm_handle_failed_procs(MPID_Group *new_failed_procs)
COMM_FOREACH(comm) {
/* if this comm is already collectively inactive and
anysources are disabled, there's no need to check */
if (!comm->ch.anysource_enabled)
if (!comm->dev.anysource_enabled)
continue;
mpi_errno = nonempty_intersection(comm, new_failed_procs, &flag);
......@@ -317,7 +317,7 @@ int MPIDI_CH3I_Comm_handle_failed_procs(MPID_Group *new_failed_procs)
MPIU_DBG_MSG_FMT(CH3_OTHER, VERBOSE,
(MPIU_DBG_FDEST, "disabling AS on communicator %p (%#08x)",
comm, comm->handle));
comm->ch.anysource_enabled = FALSE;
comm->dev.anysource_enabled = FALSE;
}
}
......
......@@ -26,7 +26,7 @@ int MPID_Comm_failure_ack(MPID_Comm *comm_ptr)
/* Update the marker for the last known failed process in this
* communciator. */
comm_ptr->ch.last_ack_rank = MPIDI_last_known_failed;
comm_ptr->dev.last_ack_rank = MPIDI_last_known_failed;
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_COMM_FAILURE_ACK);
......@@ -49,7 +49,7 @@ int MPID_Comm_failure_get_acked(MPID_Comm *comm_ptr, MPID_Group **group_ptr)
/* Get the group of all failed processes */
MPIDI_CH3U_Check_for_failed_procs();
MPIDI_CH3U_Get_failed_group(comm_ptr->ch.last_ack_rank, &failed_group);
MPIDI_CH3U_Get_failed_group(comm_ptr->dev.last_ack_rank, &failed_group);
if (failed_group == MPID_Group_empty) {
*group_ptr = MPID_Group_empty;
goto fn_exit;
......@@ -106,7 +106,7 @@ int MPID_Comm_failed_bitarray(MPID_Comm *comm_ptr, uint32_t **bitarray, int acke
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
if (acked)
MPIDI_CH3U_Get_failed_group(comm_ptr->ch.last_ack_rank, &failed_group);
MPIDI_CH3U_Get_failed_group(comm_ptr->dev.last_ack_rank, &failed_group);
else
MPIDI_CH3U_Get_failed_group(-2, &failed_group);
......
......@@ -70,11 +70,11 @@ int MPID_Comm_revoke(MPID_Comm *comm_ptr, int is_remote)
/* Start a counter to track how many revoke messages we've received from
* other ranks */
comm_ptr->ch.waiting_for_revoke = comm_ptr->local_size - 1 - is_remote - failed; /* Subtract the processes who already know about the revoke */
MPIU_DBG_MSG_FMT(CH3_OTHER, VERBOSE, (MPIU_DBG_FDEST, "Comm %08x waiting_for_revoke: %d", comm_ptr->handle, comm_ptr->ch.waiting_for_revoke));
comm_ptr->dev.waiting_for_revoke = comm_ptr->local_size - 1 - is_remote - failed; /* Subtract the processes who already know about the revoke */
MPIU_DBG_MSG_FMT(CH3_OTHER, VERBOSE, (MPIU_DBG_FDEST, "Comm %08x waiting_for_revoke: %d", comm_ptr->handle, comm_ptr->dev.waiting_for_revoke));
/* Check to see if we are done revoking */
if (comm_ptr->ch.waiting_for_revoke == 0) {
if (comm_ptr->dev.waiting_for_revoke == 0) {
MPIR_Comm_release(comm_ptr, 0);
}
......@@ -89,11 +89,11 @@ int MPID_Comm_revoke(MPID_Comm *comm_ptr, int is_remote)
MPIU_THREAD_CS_EXIT(MSGQUEUE,);
} else if (is_remote) { /* If this is local, we've already revoked and don't need to do it again. */
/* Decrement the revoke counter */
comm_ptr->ch.waiting_for_revoke--;
MPIU_DBG_MSG_FMT(CH3_OTHER, VERBOSE, (MPIU_DBG_FDEST, "Comm %08x waiting_for_revoke: %d", comm_ptr->handle, comm_ptr->ch.waiting_for_revoke));
comm_ptr->dev.waiting_for_revoke--;
MPIU_DBG_MSG_FMT(CH3_OTHER, VERBOSE, (MPIU_DBG_FDEST, "Comm %08x waiting_for_revoke: %d", comm_ptr->handle, comm_ptr->dev.waiting_for_revoke));
/* Check to see if we are done revoking */
if (comm_ptr->ch.waiting_for_revoke == 0) {
if (comm_ptr->dev.waiting_for_revoke == 0) {
MPIR_Comm_release(comm_ptr, 0);
}
}
......
......@@ -70,7 +70,7 @@ static int set_eager_threshold(MPID_Comm *comm_ptr, MPID_Info *info, void *state
MPID_MPI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_SET_EAGER_THRESHOLD);
comm_ptr->ch.eager_max_msg_sz = strtol(info->value, &endptr, 0);
comm_ptr->dev.eager_max_msg_sz = strtol(info->value, &endptr, 0);
MPIU_ERR_CHKANDJUMP1(*endptr, mpi_errno, MPI_ERR_ARG,
"**infohintparse", "**infohintparse %s",
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment