Commit 5b163672 authored by Francois Tessier's avatar Francois Tessier

Running HACC-IO on Cooley

parent 77f66a90
......@@ -17,6 +17,8 @@
- Reuse aggregators? Does not work (wrong data read) with HACC-IO, SSF, SoA, while the AoS case works
- [Cooley] 1D-array with the MEMORY_AWARE strategy fails because of a wrong memory type.
- Isolate the three features
* Random placement
......
......@@ -3,97 +3,66 @@
NODES=8
PPN=12
NPROCS=$((NODES*PPN))
TARGET="/projects/visualization/ftessier/ICS"
TARGET="/projects/Performance/ftessier/TAPICOA"
DDT="/soft/debuggers/ddt/bin/ddt --connect"
SUBFILING=FPN
cd $HOME/install/$ARCHI/bin/
export TAPIOCA_STRATEGY=TOPOLOGY_AWARE
export TAPIOCA_NBAGGR=2
export TAPIOCA_NBBUFFERS=2
export TAPIOCA_BUFFERSIZE=8388608
export TAPIOCA_AGGRTIER=DDR
export TAPIOCA_DEVNULL=false
export TAPIOCA_COMMSPLIT=true
export TAPIOCA_STRATEGY=TOPOLOGY_AWARE
export TAPIOCA_NBAGGR=1
export TAPIOCA_BUFFERSIZE=4194304 #8388608
export TAPIOCA_PIPELINING=true
export TAPIOCA_REELECTAGGR=false
# export PAMID_VERBOSE=1
# export AMID_COLLECTIVES_MEMORY_OPTIMIZED=1
# export PAMID_COLLECTIVES=1
export TAPIOCA_REELECTAGGR=true
export MPIP="-o -k 3"
function updateSettings()
{
printenv | egrep "TAPIOCA_"
SETTINGS="--env TAPIOCA_DEVNULL=$TAPIOCA_DEVNULL"
SETTINGS="$SETTINGS --env TAPIOCA_COMMSPLIT=$TAPIOCA_COMMSPLIT"
SETTINGS="$SETTINGS --env TAPIOCA_STRATEGY=$TAPIOCA_STRATEGY"
SETTINGS="--env TAPIOCA_STRATEGY=$TAPIOCA_STRATEGY"
SETTINGS="$SETTINGS --env TAPIOCA_NBAGGR=$TAPIOCA_NBAGGR"
SETTINGS="$SETTINGS --env TAPIOCA_NBBUFFERS=$TAPIOCA_NBBUFFERS"
SETTINGS="$SETTINGS --env TAPIOCA_BUFFERSIZE=$TAPIOCA_BUFFERSIZE"
SETTINGS="$SETTINGS --env TAPIOCA_AGGRTIER=$TAPIOCA_AGGRTIER"
SETTINGS="$SETTINGS --env TAPIOCA_DEVNULL=$TAPIOCA_DEVNULL"
SETTINGS="$SETTINGS --env TAPIOCA_COMMSPLIT=$TAPIOCA_COMMSPLIT"
SETTINGS="$SETTINGS --env TAPIOCA_PIPELINING=$TAPIOCA_PIPELINING"
SETTINGS="$SETTINGS --env TAPIOCA_REELECTAGGR=$TAPIOCA_REELECTAGGR"
#SETTINGS="$SETTINGS --env PAMID_VERBOSE=1"
#SETTINGS="$SETTINGS --env PAMID_COLLECTIVES_MEMORY_OPTIMIZED=1"
#SETTINGS="$SETTINGS --env PAMID_COLLECTIVES=1"
# OPTIMIZATIONS
SETTINGS="$SETTINGS --env PAMID_VERBOSE=1"
SETTINGS="$SETTINGS --env PAMID_COLLECTIVES_MEMORY_OPTIMIZED=1"
SETTINGS="$SETTINGS --env PAMID_COLLECTIVES=1"
SETTINGS="$SETTINGS --env GPFSMPIO_BALANCECONTIG=1"
SETTINGS="$SETTINGS --env GPFSMPIO_P2PCONTIG=1"
#SETTINGS="$SETTINGS --env EZTRACE_TRACE=mpi"
#SETTINGS="$SETTINGS --env EZTRACE_TRACE_DIR=/home/ftessier/log"
#SETTINGS="$SETTINGS --env LD_PRELOAD=/home/ftessier/install/linux-rhel_6-x86_64/lib/libdarshan.so"
echo $SETTINGS
}
function outputFileMPIIO ()
{
OUTPUT="$HOME/xp_tapioca/paper/Workflow/"
#OUTPUT="$HOME/xp_tapioca/debug/"
OUTPUT="${OUTPUT}/HACC_COOLEY_${BIN}_${NODES}_${PPN}"
OUTPUT="${OUTPUT}_MPIIO_DDR_to_HDD_${SUBFILING}_${PARTICLES}"
}
function outputFileTAPIOCA ()
{
#OUTPUT="$HOME/xp_tapioca/paper/Workflow/"
OUTPUT="$HOME/xp_tapioca/debug/"
OUTPUT="${OUTPUT}/HACC_COOLEY_${BIN}_${NODES}_${PPN}"
OUTPUT="${OUTPUT}_${TAPIOCA_STRATEGY}_${TAPIOCA_NBAGGR}"
OUTPUT="${OUTPUT}_${TAPIOCA_BUFFERSIZE}_DDR_to_HDD_${SUBFILING}_${PARTICLES}"
}
PARTICLES=25000
for RUN in {0..0}
do
# for TAPIOCA_BUFFERSIZE in 8388608 16777216 33554432
# do
#rm $TARGET/*
updateSettings
# for BIN in miniHACC-AoS-MPIIO-W miniHACC-AoS-MPIIO-R
# do
# outputFileMPIIO
# mpirun -f $COBALT_NODEFILE -n $NPROCS $BIN $PARTICLES >> $OUTPUT
# sleep 3
# done
updateSettings
rm $TARGET/*
rm $TARGET/*
for BIN in miniHACC-AoS-Tapioca-W #miniHACC-AoS-Tapioca-R
for LAYOUT in AOS
do
for SUB in SSF
do
outputFileTAPIOCA
mpirun --env $SETTINGS -f $COBALT_NODEFILE -n $NPROCS $BIN $PARTICLES >> $OUTPUT
sleep 2
#mpirun -f $COBALT_NODEFILE -n $NODES ls -l /scratch
sleep 2
for BIN in HACC-IO-Tapioca-W HACC-IO-Tapioca-R HACC-IO-Tapioca-WR HACC-IO-MPIIO-W HACC-IO-MPIIO-R HACC-IO-MPIIO-WR
do
mpirun --env $SETTINGS -f $COBALT_NODEFILE -n $NPROCS $BIN -p $PARTICLES -o $TARGET -s $SUB -l $LAYOUT
echo
echo "---------------------------------"
echo
sleep 5
done
done
#mpirun -f $COBALT_NODEFILE -n $NODES rm /scratch/tmp0.agg
echo "================= $NODES $RUN ================="
done
#done
\ No newline at end of file
done
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment