diff --git a/examples/HACC-IO/Makefile.bgq b/examples/HACC-IO/Makefile.bgq index b11163d163bd1a0683b137c384f2460d54fb9c8a..bdac8281706fa82c8089d1181b51c33b8ab0287c 100644 --- a/examples/HACC-IO/Makefile.bgq +++ b/examples/HACC-IO/Makefile.bgq @@ -1,6 +1,6 @@ MPICXX = mpicxx -MPI_CFLAGS = -g -O3 -I$(HOME)/install_bgq/include/ -DBGQ #-DDEBUG +MPI_CFLAGS = -g -O3 -I$(HOME)/install_bgq/include/ -DBGQ -DDEBUG MPI_LDFLAGS = -L$(HOME)/install_bgq/lib/ -ltapioca all: miniHACC-AoS miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO diff --git a/examples/HACC-IO/run_check_bgq.sh b/examples/HACC-IO/run_check_bgq.sh index 08b5f856abb775e6bd732666483a0cd89ecea313..075cbd48a5282ee58a5208296200fa6c1bc8deff 100755 --- a/examples/HACC-IO/run_check_bgq.sh +++ b/examples/HACC-IO/run_check_bgq.sh @@ -1,7 +1,6 @@ #!/bin/bash -VARS="PAMID_VERBOSE=1 BG_SHAREDMEMSIZE=64 PAMID_COLLECTIVES=1 BGLOCKLESSMPIO_F_TYPE=0x47504653 PAMID_COLLECTIVES_MEMORY_OPTIMIZED=1" -NODES=256 +NODES=128 PPN=16 NPROCS=$((NODES*PPN)) TARGET="/projects/visualization/ftessier/debug" @@ -13,6 +12,7 @@ export TAPIOCA_COMMSPLIT=true export TAPIOCA_STRATEGY=TOPOLOGY_AWARE export TAPIOCA_NBAGGR=8 export TAPIOCA_BUFFERSIZE=16777216 +export TAPIOCA_PIPELINING=true function updateSettings() { @@ -22,70 +22,82 @@ function updateSettings() SETTINGS="$SETTINGS TAPIOCA_STRATEGY=$TAPIOCA_STRATEGY" SETTINGS="$SETTINGS TAPIOCA_NBAGGR=$TAPIOCA_NBAGGR" SETTINGS="$SETTINGS TAPIOCA_BUFFERSIZE=$TAPIOCA_BUFFERSIZE" + SETTINGS="$SETTINGS TAPIOCA_PIPELINING=$TAPIOCA_PIPELINING" + + SETTINGS="$SETTINGS PAMID_VERBOSE=1" + SETTINGS="$SETTINGS PAMID_COLLECTIVES_MEMORY_OPTIMIZED=1" + SETTINGS="$SETTINGS PAMID_COLLECTIVES=1" + SETTINGS="$SETTINGS BG_SHAREDMEMSIZE=64" + SETTINGS="$SETTINGS BGLOCKLESSMPIO_F_TYPE=0x47504653" } -######################### -# Array of Structures -######################### rm $TARGET/* updateSettings -runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-AoS-MPIIO 42 -sleep 4 -let j=0 -for i in $TARGET/* -do - CONTROLAOS[$j]=`md5sum $i | cut -d ' ' -f1` - echo ${CONTROLAOS[$j]} - let j=$j+1 -done +runjob --block $COBALT_PARTNAME --envs $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-AoS 25000 1 -rm $TARGET/* -updateSettings -runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-AoS 42 -sleep 4 -let j=0 -for i in $TARGET/* -do - HASH[$j]=`md5sum $i | cut -d ' ' -f1` - if [ ${CONTROLAOS[$j]} == ${HASH[$j]} ] - then - echo -e "\e[32m[PASSED]\e[39m ${HASH[$j]}" - else - echo -e "\e[31m[FAILED]\e[39m ${HASH[$j]}" - fi - let j=$j+1 -done -######################### -# Structure of Arrays -######################### -rm $TARGET/* -updateSettings -runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-SoA-MPIIO 42 -sleep 4 -let j=0 -for i in $TARGET/* -do - CONTROLSOA[$j]=`md5sum $i | cut -d ' ' -f1` - echo ${CONTROLSOA[$j]} - let j=$j+1 -done +# ######################### +# # Array of Structures +# ######################### +# rm $TARGET/* +# updateSettings +# runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-AoS-MPIIO 42 +# sleep 4 +# let j=0 +# for i in $TARGET/* +# do +# CONTROLAOS[$j]=`md5sum $i | cut -d ' ' -f1` +# echo ${CONTROLAOS[$j]} +# let j=$j+1 +# done -rm $TARGET/* -updateSettings -runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-SoA 42 -sleep 4 -let j=0 -for i in $TARGET/* -do - HASH[$j]=`md5sum $i | cut -d ' ' -f1` - if [ ${CONTROLSOA[$j]} == ${HASH[$j]} ] - then - echo -e "\e[32m[PASSED]\e[39m ${HASH[$j]}" - else - echo -e "\e[31m[FAILED]\e[39m ${HASH[$j]}" - fi - let j=$j+1 -done +# rm $TARGET/* +# updateSettings +# runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-AoS 42 +# sleep 4 +# let j=0 +# for i in $TARGET/* +# do +# HASH[$j]=`md5sum $i | cut -d ' ' -f1` +# if [ ${CONTROLAOS[$j]} == ${HASH[$j]} ] +# then +# echo -e "\e[32m[PASSED]\e[39m ${HASH[$j]}" +# else +# echo -e "\e[31m[FAILED]\e[39m ${HASH[$j]}" +# fi +# let j=$j+1 +# done + +# ######################### +# # Structure of Arrays +# ######################### +# rm $TARGET/* +# updateSettings +# runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-SoA-MPIIO 42 +# sleep 4 +# let j=0 +# for i in $TARGET/* +# do +# CONTROLSOA[$j]=`md5sum $i | cut -d ' ' -f1` +# echo ${CONTROLSOA[$j]} +# let j=$j+1 +# done + +# rm $TARGET/* +# updateSettings +# runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-SoA 42 +# sleep 4 +# let j=0 +# for i in $TARGET/* +# do +# HASH[$j]=`md5sum $i | cut -d ' ' -f1` +# if [ ${CONTROLSOA[$j]} == ${HASH[$j]} ] +# then +# echo -e "\e[32m[PASSED]\e[39m ${HASH[$j]}" +# else +# echo -e "\e[31m[FAILED]\e[39m ${HASH[$j]}" +# fi +# let j=$j+1 +# done diff --git a/make.inc/Makefile.bgq b/make.inc/Makefile.bgq index 6784f4bb52fe90ad15f9e64e6d99c5f12973c521..26499be4581d2b6381c6979ca376243521acae78 100644 --- a/make.inc/Makefile.bgq +++ b/make.inc/Makefile.bgq @@ -1,7 +1,7 @@ -MPICXX = mpicxx +MPICXX = mpixlcxx MPI_CFLAGS = -g -O3 -I./topology/ MPI_CFLAGS += -DBGQ -DDEBUG #-DTIMING -MPI_CFLAGS += -I/bgsys/drivers/ppcfloor -I/bgsys/drivers/ppcfloor/spi/include/kernel/cnk +#MPI_CFLAGS += -I/bgsys/drivers/ppcfloor -I/bgsys/drivers/ppcfloor/spi/include/kernel/cnk INSTALL_PATH = $(HOME)/install_bgq diff --git a/tapioca.cpp b/tapioca.cpp index 4dedc9b110e28e763017494446e86c168ad3ec3d..6812aa7e598fff57a277c654799ff1df411f953c 100644 --- a/tapioca.cpp +++ b/tapioca.cpp @@ -552,8 +552,10 @@ int Tapioca::RankTopologyAware (MPI_Comm aggrComm, int64_t dataSize) // fprintf (stdout, "%d ", IOnodesList[i]); // fprintf (stdout, "\n"); // } - - //aggrCost.cost += topology.DistanceToIONode ( worldRank ) * LATENCY + (double)aggregatedData / BANDWIDTH; + +#ifdef BGQ + aggrCost.cost += topology.DistanceToIONode ( worldRank ) * LATENCY + (double)aggregatedData / BANDWIDTH; +#endif if ( this->excludedNode[this->intCoords_] ) aggrCost.cost = DBL_MAX;