Add I/O node awareness on BGQ for aggregators placement. Move to XL compiler.

examples/HACC-IO/Makefile.bgq

 MPICXX = mpicxx
 
-MPI_CFLAGS  = -g -O3 -I$(HOME)/install_bgq/include/ -DBGQ #-DDEBUG
+MPI_CFLAGS  = -g -O3 -I$(HOME)/install_bgq/include/ -DBGQ -DDEBUG
 MPI_LDFLAGS = -L$(HOME)/install_bgq/lib/ -ltapioca
 
 all: miniHACC-AoS miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO

examples/HACC-IO/run_check_bgq.sh

 #!/bin/bash
 
-VARS="PAMID_VERBOSE=1 BG_SHAREDMEMSIZE=64 PAMID_COLLECTIVES=1 BGLOCKLESSMPIO_F_TYPE=0x47504653 PAMID_COLLECTIVES_MEMORY_OPTIMIZED=1"
-NODES=256
+NODES=128
 PPN=16
 NPROCS=$((NODES*PPN))
 TARGET="/projects/visualization/ftessier/debug"
@@ -13,6 +12,7 @@ export TAPIOCA_COMMSPLIT=true
 export TAPIOCA_STRATEGY=TOPOLOGY_AWARE
 export TAPIOCA_NBAGGR=8
 export TAPIOCA_BUFFERSIZE=16777216
+export TAPIOCA_PIPELINING=true
 
 function updateSettings()
 {
@@ -22,70 +22,82 @@ function updateSettings()
     SETTINGS="$SETTINGS TAPIOCA_STRATEGY=$TAPIOCA_STRATEGY"
     SETTINGS="$SETTINGS TAPIOCA_NBAGGR=$TAPIOCA_NBAGGR"
     SETTINGS="$SETTINGS TAPIOCA_BUFFERSIZE=$TAPIOCA_BUFFERSIZE"
+    SETTINGS="$SETTINGS TAPIOCA_PIPELINING=$TAPIOCA_PIPELINING"
+    
+    SETTINGS="$SETTINGS PAMID_VERBOSE=1"
+    SETTINGS="$SETTINGS PAMID_COLLECTIVES_MEMORY_OPTIMIZED=1"
+    SETTINGS="$SETTINGS PAMID_COLLECTIVES=1"
+    SETTINGS="$SETTINGS BG_SHAREDMEMSIZE=64"
+    SETTINGS="$SETTINGS BGLOCKLESSMPIO_F_TYPE=0x47504653"
 }
 
-#########################
-# Array of Structures
-#########################
 rm $TARGET/*
 updateSettings
-runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-AoS-MPIIO 42
-sleep 4
-let j=0
-for i in $TARGET/*
-do
-    CONTROLAOS[$j]=`md5sum $i | cut -d ' ' -f1`
-    echo ${CONTROLAOS[$j]}
-    let j=$j+1
-done
+runjob --block $COBALT_PARTNAME --envs $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-AoS 25000 1
 
-rm $TARGET/*
-updateSettings
-runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-AoS 42
-sleep 4
-let j=0
-for i in $TARGET/*
-do
-    HASH[$j]=`md5sum $i | cut -d ' ' -f1`
-    if [ ${CONTROLAOS[$j]} == ${HASH[$j]} ]
-    then
-        echo -e "\e[32m[PASSED]\e[39m ${HASH[$j]}"
-    else
-        echo -e "\e[31m[FAILED]\e[39m ${HASH[$j]}"
-    fi
-    let j=$j+1
-done
 
-#########################
-# Structure of Arrays
-#########################
-rm $TARGET/*
-updateSettings
-runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-SoA-MPIIO 42
-sleep 4
-let j=0
-for i in $TARGET/*
-do
-    CONTROLSOA[$j]=`md5sum $i | cut -d ' ' -f1`
-    echo ${CONTROLSOA[$j]}
-    let j=$j+1
-done
+# #########################
+# # Array of Structures
+# #########################
+# rm $TARGET/*
+# updateSettings
+# runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-AoS-MPIIO 42
+# sleep 4
+# let j=0
+# for i in $TARGET/*
+# do
+#     CONTROLAOS[$j]=`md5sum $i | cut -d ' ' -f1`
+#     echo ${CONTROLAOS[$j]}
+#     let j=$j+1
+# done
 
-rm $TARGET/*
-updateSettings
-runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-SoA 42
-sleep 4
-let j=0
-for i in $TARGET/*
-do
-    HASH[$j]=`md5sum $i | cut -d ' ' -f1`
-    if [ ${CONTROLSOA[$j]} == ${HASH[$j]} ]
-    then
-        echo -e "\e[32m[PASSED]\e[39m ${HASH[$j]}"
-    else
-        echo -e "\e[31m[FAILED]\e[39m ${HASH[$j]}"
-    fi
-    let j=$j+1
-done
+# rm $TARGET/*
+# updateSettings
+# runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-AoS 42
+# sleep 4
+# let j=0
+# for i in $TARGET/*
+# do
+#     HASH[$j]=`md5sum $i | cut -d ' ' -f1`
+#     if [ ${CONTROLAOS[$j]} == ${HASH[$j]} ]
+#     then
+#         echo -e "\e[32m[PASSED]\e[39m ${HASH[$j]}"
+#     else
+#         echo -e "\e[31m[FAILED]\e[39m ${HASH[$j]}"
+#     fi
+#     let j=$j+1
+# done
+
+# #########################
+# # Structure of Arrays
+# #########################
+# rm $TARGET/*
+# updateSettings
+# runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-SoA-MPIIO 42
+# sleep 4
+# let j=0
+# for i in $TARGET/*
+# do
+#     CONTROLSOA[$j]=`md5sum $i | cut -d ' ' -f1`
+#     echo ${CONTROLSOA[$j]}
+#     let j=$j+1
+# done
+
+# rm $TARGET/*
+# updateSettings
+# runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-SoA 42
+# sleep 4
+# let j=0
+# for i in $TARGET/*
+# do
+#     HASH[$j]=`md5sum $i | cut -d ' ' -f1`
+#     if [ ${CONTROLSOA[$j]} == ${HASH[$j]} ]
+#     then
+#         echo -e "\e[32m[PASSED]\e[39m ${HASH[$j]}"
+#     else
+#         echo -e "\e[31m[FAILED]\e[39m ${HASH[$j]}"
+#     fi
+#     let j=$j+1
+# done
 
 

make.inc/Makefile.bgq

-MPICXX = mpicxx
+MPICXX = mpixlcxx
 
 MPI_CFLAGS   = -g -O3 -I./topology/
 MPI_CFLAGS  += -DBGQ -DDEBUG #-DTIMING                                                                                                                                              
-MPI_CFLAGS  += -I/bgsys/drivers/ppcfloor -I/bgsys/drivers/ppcfloor/spi/include/kernel/cnk
+#MPI_CFLAGS  += -I/bgsys/drivers/ppcfloor -I/bgsys/drivers/ppcfloor/spi/include/kernel/cnk
 
 INSTALL_PATH  = $(HOME)/install_bgq

tapioca.cpp

@@ -552,8 +552,10 @@ int Tapioca::RankTopologyAware (MPI_Comm aggrComm, int64_t dataSize)
   //     fprintf (stdout, "%d ", IOnodesList[i]);
   //   fprintf (stdout, "\n");
   // }
-    
-  //aggrCost.cost += topology.DistanceToIONode ( worldRank ) * LATENCY + (double)aggregatedData / BANDWIDTH;
+
+#ifdef BGQ
+  aggrCost.cost += topology.DistanceToIONode ( worldRank ) * LATENCY + (double)aggregatedData / BANDWIDTH;
+#endif
 
   if ( this->excludedNode[this->intCoords_] )
     aggrCost.cost = DBL_MAX;