From a943b3b867a40b2751bbf839d509f6e6509ab2c9 Mon Sep 17 00:00:00 2001
From: Francois Tessier <ftessier@anl.gov>
Date: Sun, 30 Apr 2017 04:49:50 +0000
Subject: [PATCH] Add I/O node awareness on BGQ for aggregators placement. Move
 to XL compiler.

---
 examples/HACC-IO/Makefile.bgq     |   2 +-
 examples/HACC-IO/run_check_bgq.sh | 132 ++++++++++++++++--------------
 make.inc/Makefile.bgq             |   4 +-
 tapioca.cpp                       |   6 +-
 4 files changed, 79 insertions(+), 65 deletions(-)

diff --git a/examples/HACC-IO/Makefile.bgq b/examples/HACC-IO/Makefile.bgq
index b11163d..bdac828 100644
--- a/examples/HACC-IO/Makefile.bgq
+++ b/examples/HACC-IO/Makefile.bgq
@@ -1,6 +1,6 @@
 MPICXX = mpicxx
 
-MPI_CFLAGS  = -g -O3 -I$(HOME)/install_bgq/include/ -DBGQ #-DDEBUG
+MPI_CFLAGS  = -g -O3 -I$(HOME)/install_bgq/include/ -DBGQ -DDEBUG
 MPI_LDFLAGS = -L$(HOME)/install_bgq/lib/ -ltapioca
 
 all: miniHACC-AoS miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO
diff --git a/examples/HACC-IO/run_check_bgq.sh b/examples/HACC-IO/run_check_bgq.sh
index 08b5f85..075cbd4 100755
--- a/examples/HACC-IO/run_check_bgq.sh
+++ b/examples/HACC-IO/run_check_bgq.sh
@@ -1,7 +1,6 @@
 #!/bin/bash
 
-VARS="PAMID_VERBOSE=1 BG_SHAREDMEMSIZE=64 PAMID_COLLECTIVES=1 BGLOCKLESSMPIO_F_TYPE=0x47504653 PAMID_COLLECTIVES_MEMORY_OPTIMIZED=1"
-NODES=256
+NODES=128
 PPN=16
 NPROCS=$((NODES*PPN))
 TARGET="/projects/visualization/ftessier/debug"
@@ -13,6 +12,7 @@ export TAPIOCA_COMMSPLIT=true
 export TAPIOCA_STRATEGY=TOPOLOGY_AWARE
 export TAPIOCA_NBAGGR=8
 export TAPIOCA_BUFFERSIZE=16777216
+export TAPIOCA_PIPELINING=true
 
 function updateSettings()
 {
@@ -22,70 +22,82 @@ function updateSettings()
     SETTINGS="$SETTINGS TAPIOCA_STRATEGY=$TAPIOCA_STRATEGY"
     SETTINGS="$SETTINGS TAPIOCA_NBAGGR=$TAPIOCA_NBAGGR"
     SETTINGS="$SETTINGS TAPIOCA_BUFFERSIZE=$TAPIOCA_BUFFERSIZE"
+    SETTINGS="$SETTINGS TAPIOCA_PIPELINING=$TAPIOCA_PIPELINING"
+    
+    SETTINGS="$SETTINGS PAMID_VERBOSE=1"
+    SETTINGS="$SETTINGS PAMID_COLLECTIVES_MEMORY_OPTIMIZED=1"
+    SETTINGS="$SETTINGS PAMID_COLLECTIVES=1"
+    SETTINGS="$SETTINGS BG_SHAREDMEMSIZE=64"
+    SETTINGS="$SETTINGS BGLOCKLESSMPIO_F_TYPE=0x47504653"
 }
 
-#########################
-# Array of Structures
-#########################
 rm $TARGET/*
 updateSettings
-runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-AoS-MPIIO 42
-sleep 4
-let j=0
-for i in $TARGET/*
-do
-    CONTROLAOS[$j]=`md5sum $i | cut -d ' ' -f1`
-    echo ${CONTROLAOS[$j]}
-    let j=$j+1
-done
+runjob --block $COBALT_PARTNAME --envs $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-AoS 25000 1
 
-rm $TARGET/*
-updateSettings
-runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-AoS 42
-sleep 4
-let j=0
-for i in $TARGET/*
-do
-    HASH[$j]=`md5sum $i | cut -d ' ' -f1`
-    if [ ${CONTROLAOS[$j]} == ${HASH[$j]} ]
-    then
-        echo -e "\e[32m[PASSED]\e[39m ${HASH[$j]}"
-    else
-        echo -e "\e[31m[FAILED]\e[39m ${HASH[$j]}"
-    fi
-    let j=$j+1
-done
 
-#########################
-# Structure of Arrays
-#########################
-rm $TARGET/*
-updateSettings
-runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-SoA-MPIIO 42
-sleep 4
-let j=0
-for i in $TARGET/*
-do
-    CONTROLSOA[$j]=`md5sum $i | cut -d ' ' -f1`
-    echo ${CONTROLSOA[$j]}
-    let j=$j+1
-done
+# #########################
+# # Array of Structures
+# #########################
+# rm $TARGET/*
+# updateSettings
+# runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-AoS-MPIIO 42
+# sleep 4
+# let j=0
+# for i in $TARGET/*
+# do
+#     CONTROLAOS[$j]=`md5sum $i | cut -d ' ' -f1`
+#     echo ${CONTROLAOS[$j]}
+#     let j=$j+1
+# done
 
-rm $TARGET/*
-updateSettings
-runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-SoA 42
-sleep 4
-let j=0
-for i in $TARGET/*
-do
-    HASH[$j]=`md5sum $i | cut -d ' ' -f1`
-    if [ ${CONTROLSOA[$j]} == ${HASH[$j]} ]
-    then
-        echo -e "\e[32m[PASSED]\e[39m ${HASH[$j]}"
-    else
-        echo -e "\e[31m[FAILED]\e[39m ${HASH[$j]}"
-    fi
-    let j=$j+1
-done
+# rm $TARGET/*
+# updateSettings
+# runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-AoS 42
+# sleep 4
+# let j=0
+# for i in $TARGET/*
+# do
+#     HASH[$j]=`md5sum $i | cut -d ' ' -f1`
+#     if [ ${CONTROLAOS[$j]} == ${HASH[$j]} ]
+#     then
+#         echo -e "\e[32m[PASSED]\e[39m ${HASH[$j]}"
+#     else
+#         echo -e "\e[31m[FAILED]\e[39m ${HASH[$j]}"
+#     fi
+#     let j=$j+1
+# done
+
+# #########################
+# # Structure of Arrays
+# #########################
+# rm $TARGET/*
+# updateSettings
+# runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-SoA-MPIIO 42
+# sleep 4
+# let j=0
+# for i in $TARGET/*
+# do
+#     CONTROLSOA[$j]=`md5sum $i | cut -d ' ' -f1`
+#     echo ${CONTROLSOA[$j]}
+#     let j=$j+1
+# done
+
+# rm $TARGET/*
+# updateSettings
+# runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-SoA 42
+# sleep 4
+# let j=0
+# for i in $TARGET/*
+# do
+#     HASH[$j]=`md5sum $i | cut -d ' ' -f1`
+#     if [ ${CONTROLSOA[$j]} == ${HASH[$j]} ]
+#     then
+#         echo -e "\e[32m[PASSED]\e[39m ${HASH[$j]}"
+#     else
+#         echo -e "\e[31m[FAILED]\e[39m ${HASH[$j]}"
+#     fi
+#     let j=$j+1
+# done
 
 
diff --git a/make.inc/Makefile.bgq b/make.inc/Makefile.bgq
index 6784f4b..26499be 100644
--- a/make.inc/Makefile.bgq
+++ b/make.inc/Makefile.bgq
@@ -1,7 +1,7 @@
-MPICXX = mpicxx
+MPICXX = mpixlcxx
 
 MPI_CFLAGS   = -g -O3 -I./topology/
 MPI_CFLAGS  += -DBGQ -DDEBUG #-DTIMING                                                                                                                                              
-MPI_CFLAGS  += -I/bgsys/drivers/ppcfloor -I/bgsys/drivers/ppcfloor/spi/include/kernel/cnk
+#MPI_CFLAGS  += -I/bgsys/drivers/ppcfloor -I/bgsys/drivers/ppcfloor/spi/include/kernel/cnk
 
 INSTALL_PATH  = $(HOME)/install_bgq
diff --git a/tapioca.cpp b/tapioca.cpp
index 4dedc9b..6812aa7 100644
--- a/tapioca.cpp
+++ b/tapioca.cpp
@@ -552,8 +552,10 @@ int Tapioca::RankTopologyAware (MPI_Comm aggrComm, int64_t dataSize)
   //     fprintf (stdout, "%d ", IOnodesList[i]);
   //   fprintf (stdout, "\n");
   // }
-    
-  //aggrCost.cost += topology.DistanceToIONode ( worldRank ) * LATENCY + (double)aggregatedData / BANDWIDTH;
+
+#ifdef BGQ
+  aggrCost.cost += topology.DistanceToIONode ( worldRank ) * LATENCY + (double)aggregatedData / BANDWIDTH;
+#endif
 
   if ( this->excludedNode[this->intCoords_] )
     aggrCost.cost = DBL_MAX;
-- 
2.26.2