From a943b3b867a40b2751bbf839d509f6e6509ab2c9 Mon Sep 17 00:00:00 2001 From: Francois Tessier Date: Sun, 30 Apr 2017 04:49:50 +0000 Subject: [PATCH] Add I/O node awareness on BGQ for aggregators placement. Move to XL compiler. --- examples/HACC-IO/Makefile.bgq | 2 +- examples/HACC-IO/run_check_bgq.sh | 132 ++++++++++++++++-------------- make.inc/Makefile.bgq | 4 +- tapioca.cpp | 6 +- 4 files changed, 79 insertions(+), 65 deletions(-) diff --git a/examples/HACC-IO/Makefile.bgq b/examples/HACC-IO/Makefile.bgq index b11163d..bdac828 100644 --- a/examples/HACC-IO/Makefile.bgq +++ b/examples/HACC-IO/Makefile.bgq @@ -1,6 +1,6 @@ MPICXX = mpicxx -MPI_CFLAGS = -g -O3 -I$(HOME)/install_bgq/include/ -DBGQ #-DDEBUG +MPI_CFLAGS = -g -O3 -I$(HOME)/install_bgq/include/ -DBGQ -DDEBUG MPI_LDFLAGS = -L$(HOME)/install_bgq/lib/ -ltapioca all: miniHACC-AoS miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO diff --git a/examples/HACC-IO/run_check_bgq.sh b/examples/HACC-IO/run_check_bgq.sh index 08b5f85..075cbd4 100755 --- a/examples/HACC-IO/run_check_bgq.sh +++ b/examples/HACC-IO/run_check_bgq.sh @@ -1,7 +1,6 @@ #!/bin/bash -VARS="PAMID_VERBOSE=1 BG_SHAREDMEMSIZE=64 PAMID_COLLECTIVES=1 BGLOCKLESSMPIO_F_TYPE=0x47504653 PAMID_COLLECTIVES_MEMORY_OPTIMIZED=1" -NODES=256 +NODES=128 PPN=16 NPROCS=$((NODES*PPN)) TARGET="/projects/visualization/ftessier/debug" @@ -13,6 +12,7 @@ export TAPIOCA_COMMSPLIT=true export TAPIOCA_STRATEGY=TOPOLOGY_AWARE export TAPIOCA_NBAGGR=8 export TAPIOCA_BUFFERSIZE=16777216 +export TAPIOCA_PIPELINING=true function updateSettings() { @@ -22,70 +22,82 @@ function updateSettings() SETTINGS="$SETTINGS TAPIOCA_STRATEGY=$TAPIOCA_STRATEGY" SETTINGS="$SETTINGS TAPIOCA_NBAGGR=$TAPIOCA_NBAGGR" SETTINGS="$SETTINGS TAPIOCA_BUFFERSIZE=$TAPIOCA_BUFFERSIZE" + SETTINGS="$SETTINGS TAPIOCA_PIPELINING=$TAPIOCA_PIPELINING" + + SETTINGS="$SETTINGS PAMID_VERBOSE=1" + SETTINGS="$SETTINGS PAMID_COLLECTIVES_MEMORY_OPTIMIZED=1" + SETTINGS="$SETTINGS PAMID_COLLECTIVES=1" + SETTINGS="$SETTINGS BG_SHAREDMEMSIZE=64" + SETTINGS="$SETTINGS BGLOCKLESSMPIO_F_TYPE=0x47504653" } -######################### -# Array of Structures -######################### rm $TARGET/* updateSettings -runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-AoS-MPIIO 42 -sleep 4 -let j=0 -for i in $TARGET/* -do - CONTROLAOS[$j]=`md5sum $i | cut -d ' ' -f1` - echo ${CONTROLAOS[$j]} - let j=$j+1 -done +runjob --block $COBALT_PARTNAME --envs $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-AoS 25000 1 -rm $TARGET/* -updateSettings -runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-AoS 42 -sleep 4 -let j=0 -for i in $TARGET/* -do - HASH[$j]=`md5sum $i | cut -d ' ' -f1` - if [ ${CONTROLAOS[$j]} == ${HASH[$j]} ] - then - echo -e "\e[32m[PASSED]\e[39m ${HASH[$j]}" - else - echo -e "\e[31m[FAILED]\e[39m ${HASH[$j]}" - fi - let j=$j+1 -done -######################### -# Structure of Arrays -######################### -rm $TARGET/* -updateSettings -runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-SoA-MPIIO 42 -sleep 4 -let j=0 -for i in $TARGET/* -do - CONTROLSOA[$j]=`md5sum $i | cut -d ' ' -f1` - echo ${CONTROLSOA[$j]} - let j=$j+1 -done +# ######################### +# # Array of Structures +# ######################### +# rm $TARGET/* +# updateSettings +# runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-AoS-MPIIO 42 +# sleep 4 +# let j=0 +# for i in $TARGET/* +# do +# CONTROLAOS[$j]=`md5sum $i | cut -d ' ' -f1` +# echo ${CONTROLAOS[$j]} +# let j=$j+1 +# done -rm $TARGET/* -updateSettings -runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-SoA 42 -sleep 4 -let j=0 -for i in $TARGET/* -do - HASH[$j]=`md5sum $i | cut -d ' ' -f1` - if [ ${CONTROLSOA[$j]} == ${HASH[$j]} ] - then - echo -e "\e[32m[PASSED]\e[39m ${HASH[$j]}" - else - echo -e "\e[31m[FAILED]\e[39m ${HASH[$j]}" - fi - let j=$j+1 -done +# rm $TARGET/* +# updateSettings +# runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-AoS 42 +# sleep 4 +# let j=0 +# for i in $TARGET/* +# do +# HASH[$j]=`md5sum $i | cut -d ' ' -f1` +# if [ ${CONTROLAOS[$j]} == ${HASH[$j]} ] +# then +# echo -e "\e[32m[PASSED]\e[39m ${HASH[$j]}" +# else +# echo -e "\e[31m[FAILED]\e[39m ${HASH[$j]}" +# fi +# let j=$j+1 +# done + +# ######################### +# # Structure of Arrays +# ######################### +# rm $TARGET/* +# updateSettings +# runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-SoA-MPIIO 42 +# sleep 4 +# let j=0 +# for i in $TARGET/* +# do +# CONTROLSOA[$j]=`md5sum $i | cut -d ' ' -f1` +# echo ${CONTROLSOA[$j]} +# let j=$j+1 +# done + +# rm $TARGET/* +# updateSettings +# runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-SoA 42 +# sleep 4 +# let j=0 +# for i in $TARGET/* +# do +# HASH[$j]=`md5sum $i | cut -d ' ' -f1` +# if [ ${CONTROLSOA[$j]} == ${HASH[$j]} ] +# then +# echo -e "\e[32m[PASSED]\e[39m ${HASH[$j]}" +# else +# echo -e "\e[31m[FAILED]\e[39m ${HASH[$j]}" +# fi +# let j=$j+1 +# done diff --git a/make.inc/Makefile.bgq b/make.inc/Makefile.bgq index 6784f4b..26499be 100644 --- a/make.inc/Makefile.bgq +++ b/make.inc/Makefile.bgq @@ -1,7 +1,7 @@ -MPICXX = mpicxx +MPICXX = mpixlcxx MPI_CFLAGS = -g -O3 -I./topology/ MPI_CFLAGS += -DBGQ -DDEBUG #-DTIMING -MPI_CFLAGS += -I/bgsys/drivers/ppcfloor -I/bgsys/drivers/ppcfloor/spi/include/kernel/cnk +#MPI_CFLAGS += -I/bgsys/drivers/ppcfloor -I/bgsys/drivers/ppcfloor/spi/include/kernel/cnk INSTALL_PATH = $(HOME)/install_bgq diff --git a/tapioca.cpp b/tapioca.cpp index 4dedc9b..6812aa7 100644 --- a/tapioca.cpp +++ b/tapioca.cpp @@ -552,8 +552,10 @@ int Tapioca::RankTopologyAware (MPI_Comm aggrComm, int64_t dataSize) // fprintf (stdout, "%d ", IOnodesList[i]); // fprintf (stdout, "\n"); // } - - //aggrCost.cost += topology.DistanceToIONode ( worldRank ) * LATENCY + (double)aggregatedData / BANDWIDTH; + +#ifdef BGQ + aggrCost.cost += topology.DistanceToIONode ( worldRank ) * LATENCY + (double)aggregatedData / BANDWIDTH; +#endif if ( this->excludedNode[this->intCoords_] ) aggrCost.cost = DBL_MAX; -- 2.26.2