Add DMAPP support on XC40 for one-sided communication. XP on Theta.

Makefile

-include make.inc/Makefile.bgq
+include make.inc/Makefile.xc40
 
 all:libtapioca.a
 

examples/HACC-IO/Makefile

-MPICXX = mpicxx
+MPICXX = CC
 
-MPI_CFLAGS  = -g -O3 -I$(HOME)/install_bgq/include/ -DBGQ #-DDEBUG
-MPI_LDFLAGS = -L$(HOME)/install_bgq/lib/ -ltapioca
+MPI_CFLAGS  = -g -O3 -I$(HOME)/install/include/ -DXC40 -DDEBUG
+MPI_LDFLAGS = -L$(HOME)/install/lib/ -ltapioca -llustreapi -Wl,--whole-archive,-ldmapp,--no-whole-archive
 
 all: miniHACC-AoS miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO
 

examples/HACC-IO/miniHACC-AoS-MPIIO.cpp

@@ -14,7 +14,7 @@
 int main (int argc, char * argv[])
 {
   int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank, file_id;
-  int64_t num_particles = 25000;
+  int64_t num_particles = 25005;
   int64_t sub_particles, tot_particles, particle_size, file_size, tot_size;
   int64_t scan_size = 0, offset;
   double start_time, end_time, tot_time, max_time;
@@ -23,12 +23,13 @@ int main (int argc, char * argv[])
   MPI_File file_handle;
   MPI_Status status;
   char output[100];
+  Tapioca tp;
   
   MPI_Init(&argc, &argv);
   MPI_Comm_size(MPI_COMM_WORLD, &world_numtasks);
   MPI_Comm_rank(MPI_COMM_WORLD, &world_myrank);
 
-  mycolor = MPIX_IO_link_id ();
+  mycolor = tp.topology.BridgeNodeId ();
   mykey = world_myrank;
 
   MPI_Comm_split (MPI_COMM_WORLD, mycolor, mykey, &sub_comm);

examples/HACC-IO/miniHACC-AoS.cpp

@@ -14,7 +14,7 @@
 int main (int argc, char * argv[])
 {
   int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank, i, file_id;
-  int64_t num_particles = 25000;
+  int64_t num_particles = 100000;
   int64_t sub_particles, tot_particles, particle_size, file_size, tot_size;
   int64_t scan_size = 0, offset, hdr = 0;
   double start_time, end_time, tot_time, max_time;

examples/HACC-IO/miniHACC-SoA-MPIIO.cpp

@@ -14,7 +14,7 @@
 int main (int argc, char * argv[])
 {
   int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank, file_id;
-  int64_t num_particles = 25000;
+  int64_t num_particles = 25005;
   int64_t sub_particles, tot_particles, particle_size, file_size, tot_size;
   int64_t scan_size = 0, offset;
   double start_time, end_time, tot_time, max_time;
@@ -23,12 +23,13 @@ int main (int argc, char * argv[])
   MPI_File file_handle;
   MPI_Status status;
   char output[100];
+  Tapioca tp;
   
   MPI_Init(&argc, &argv);
   MPI_Comm_size(MPI_COMM_WORLD, &world_numtasks);
   MPI_Comm_rank(MPI_COMM_WORLD, &world_myrank);
 
-  mycolor = MPIX_IO_link_id ();
+  mycolor = tp.topology.BridgeNodeId ();
   mykey = world_myrank;
 
   MPI_Comm_split (MPI_COMM_WORLD, mycolor, mykey, &sub_comm);

examples/HACC-IO/miniHACC-SoA.cpp

@@ -14,7 +14,7 @@
 int main (int argc, char * argv[])
 {
   int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank, i, file_id;
-  int64_t num_particles = 25000;
+  int64_t num_particles = 25005;
   int64_t sub_particles, tot_particles, particle_size, file_size, tot_size;
   int64_t scan_size = 0, offset, hdr = 0;
   double start_time, end_time, tot_time, max_time;

examples/HACC-IO/run_check_xc40.sh

@@ -18,18 +18,20 @@ export TAPIOCA_BUFFERSIZE=16777216
 function updateSettings()
 {
     printenv | egrep "TAPIOCA_"
-    SETTINGS="TAPIOCA_DEVNULL=$TAPIOCA_DEVNULL"
-    SETTINGS="$SETTINGS TAPIOCA_COMMSPLIT=$TAPIOCA_COMMSPLIT"
-    SETTINGS="$SETTINGS TAPIOCA_STRATEGY=$TAPIOCA_STRATEGY"
-    SETTINGS="$SETTINGS TAPIOCA_NBAGGR=$TAPIOCA_NBAGGR"
-    SETTINGS="$SETTINGS TAPIOCA_BUFFERSIZE=$TAPIOCA_BUFFERSIZE"
+    SETTINGS="-e TAPIOCA_DEVNULL=$TAPIOCA_DEVNULL"
+    SETTINGS="$SETTINGS -e TAPIOCA_COMMSPLIT=$TAPIOCA_COMMSPLIT"
+    SETTINGS="$SETTINGS -e TAPIOCA_STRATEGY=$TAPIOCA_STRATEGY"
+    SETTINGS="$SETTINGS -e TAPIOCA_NBAGGR=$TAPIOCA_NBAGGR"
+    SETTINGS="$SETTINGS -e TAPIOCA_BUFFERSIZE=$TAPIOCA_BUFFERSIZE"
+
+    SETTINGS="$SETTINGS -e MPICH_RMA_OVER_DMAPP=1"
 }
 
 
 rm $TARGET/*
 updateSettings
 
-$DDT aprun -n $NPROCS -N $PPN ./miniHACC-AoS 1
+aprun $SETTINGS -n $NPROCS -N $PPN ./miniHACC-AoS
 #########################
 # Array of Structures
 #########################

make.inc/Makefile.xc40

@@ -3,6 +3,6 @@ MPICXX = CC
 MPI_CFLAGS   = -g -O3 -I./topology/ -I./topology/include/
 MPI_CFLAGS  += -DXC40 -DDEBUG #-DTIMING
 
-MPI_LDFLAGS += -llustreapi
+MPI_LDFLAGS += -llustreapi -Wl,--whole-archive,-ldmapp,--no-whole-archive
 
 INSTALL_PATH  = $(HOME)/install

topology/cray_xc40_theta.hpp

@@ -121,6 +121,7 @@ public:
     coord[1] = xyz.mesh_y;
     coord[2] = xyz.mesh_z;
     coord[3] = nid;
+    coord[4] = sched_getcpu();
   }