[svn-r3243] added SMP-aware Reduce

src/mpi/coll/reduce.c

@@ -716,6 +716,7 @@ int MPI_Reduce(void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
     int mpi_errno = MPI_SUCCESS;
     MPID_Comm *comm_ptr = NULL;
     MPID_MPI_STATE_DECL(MPID_STATE_MPI_REDUCE);
+    MPIU_CHKLMEM_DECL(1);
 
     MPIR_ERRTEST_INITIALIZED_ORDIE();
     
@@ -832,8 +833,91 @@ int MPI_Reduce(void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
     {
         if (comm_ptr->comm_kind == MPID_INTRACOMM) {
             /* intracommunicator */
+#if USE_SMP_COLLECTIVES
+            if (MPIR_Comm_is_node_aware(comm_ptr)) {
+
+		void *tmp_buf;
+		MPI_Aint  true_lb, true_extent, extent; 
+		MPIU_THREADPRIV_DECL;
+		MPIU_THREADPRIV_GET;
+
+		/* Create a temporary buffer on local roots of all nodes */
+		if (comm_ptr->node_roots_comm != NULL) {
+
+		    MPIR_Nest_incr();
+		    mpi_errno = NMPI_Type_get_true_extent(datatype, &true_lb, &true_extent);  
+		    MPIR_Nest_decr();
+		    if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
+		    MPID_Datatype_get_extent_macro(datatype, extent);
+
+		    MPID_Ensure_Aint_fits_in_pointer(count * MPIR_MAX(extent, true_extent));
+
+		    MPIU_CHKLMEM_MALLOC(tmp_buf, void *, count*(MPIR_MAX(extent,true_extent)),
+					mpi_errno, "temporary buffer");
+		    /* adjust for potential negative lower bound in datatype */
+		    tmp_buf = (void *)((char*)tmp_buf - true_lb);
+		}    
+
+                /* do the intranode reduce on all nodes other than the root's node */
+                if (comm_ptr->node_comm != NULL &&
+                    MPIU_Get_intranode_rank(comm_ptr, root) == -1) { 
+		    mpi_errno = MPIR_Reduce(sendbuf, tmp_buf, count, datatype,
+					    op, 0, comm_ptr->node_comm);
+		    if (mpi_errno) goto fn_fail;
+                }
+
+                /* do the internode reduce to the root's node */
+                if (comm_ptr->node_roots_comm != NULL) {
+
+		    if (comm_ptr->node_roots_comm->rank != MPIU_Get_internode_rank(comm_ptr, root))
+			/* I am not on root's node */
+			mpi_errno = MPIR_Reduce(tmp_buf, NULL, count, datatype,
+					    op, MPIU_Get_internode_rank(comm_ptr, root), 
+					    comm_ptr->node_roots_comm);
+		    else { /* I am on root's node. I have not participated in the earlier reduce. */
+			if (comm_ptr->rank != root) {
+			    /* I am not the root though. I don't have a valid recvbuf.
+                               Use tmp_buf as recvbuf. */
+
+			    mpi_errno = MPIR_Reduce(sendbuf, tmp_buf, count, datatype,
+					    op, MPIU_Get_internode_rank(comm_ptr, root), 
+					    comm_ptr->node_roots_comm);
+
+			    /* point sendbuf at tmp_buf to make final intranode reduce easy */
+			    sendbuf = tmp_buf;
+			}
+			else {
+			    /* I am the root. in_place is automatically handled. */
+
+			    mpi_errno = MPIR_Reduce(sendbuf, recvbuf, count, datatype,
+					    op, MPIU_Get_internode_rank(comm_ptr, root), 
+					    comm_ptr->node_roots_comm);
+
+			    /* set sendbuf to MPI_IN_PLACE to make final intranode reduce easy. */
+			    sendbuf = MPI_IN_PLACE;
+			}
+		    }
+
+                    if (mpi_errno) goto fn_fail;
+                }
+
+                /* do the intranode reduce on the root's node */
+                if (comm_ptr->node_comm != NULL &&
+                    MPIU_Get_intranode_rank(comm_ptr, root) != -1) { 
+                    mpi_errno = MPIR_Reduce(sendbuf, recvbuf, count, datatype,
+					    op, MPIU_Get_intranode_rank(comm_ptr, root),
+					    comm_ptr->node_comm);
+                }
+
+            }
+            else {
+		mpi_errno = MPIR_Reduce(sendbuf, recvbuf, count, datatype,
+                                    op, root, comm_ptr); 
+            }
+#else
             mpi_errno = MPIR_Reduce(sendbuf, recvbuf, count, datatype,
                                     op, root, comm_ptr); 
+#endif
 	}
         else {
             /* intercommunicator */
@@ -847,6 +931,7 @@ int MPI_Reduce(void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
     /* ... end of body of routine ... */
     
   fn_exit:
+    MPIU_CHKLMEM_FREEALL();
     MPID_MPI_COLL_FUNC_EXIT(MPID_STATE_MPI_REDUCE);
     MPIU_THREAD_CS_EXIT(ALLFUNC,);
     return mpi_errno;