Skip to content

GitLab

Commit 1c90dfb0 authored by Francois Tessier's avatar Francois Tessier
Browse files

In HACC-IO, add MPI file information to set custom values for MPI I/O hints + update run scripts

parent ba106125
Hide whitespace changes
Inline Side-by-side
Showing with 44 additions and 25 deletions
examples/HACC-IO/miniHACC-AoS-MPIIO.cpp
View file @ 1c90dfb0
......@@ -21,6 +21,7 @@ int main (int argc, char * argv[])
double io_bw;
MPI_Comm sub_comm;
MPI_File file_handle;
MPI_Info info;
MPI_Status status;
char output[100];
Tapioca tp;
......@@ -105,10 +106,14 @@ int main (int argc, char * argv[])
mycolor, (double)file_size/(1024*1024), sub_particles);
}
MPI_Info_create ( &info );
MPI_Info_set ( info, "cb_nodes", getenv("TAPIOCA_NBAGGR") );
MPI_Info_set ( info, "bg_nodes_pset", getenv("TAPIOCA_NBAGGR") );
start_time = MPI_Wtime();
MPI_File_open(sub_comm, output,
MPI_MODE_WRONLY, MPI_INFO_NULL, &file_handle);
MPI_MODE_WRONLY, info, &file_handle);
offset = scan_size * particle_size;
......@@ -172,7 +177,7 @@ int main (int argc, char * argv[])
start_time = MPI_Wtime();
MPI_File_open(sub_comm, output,
MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handle);
MPI_MODE_RDONLY, info, &file_handle);
if (0 == sub_myrank)
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] Read output file\n", mycolor);
......
examples/HACC-IO/miniHACC-AoS.cpp
View file @ 1c90dfb0
......@@ -30,9 +30,10 @@ int main (int argc, char * argv[])
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &world_numtasks);
MPI_Comm_rank(MPI_COMM_WORLD, &world_myrank);
mycolor = tp.topology.BridgeNodeId ();
mykey = world_myrank;
//mycolor = tp.topology.BridgeNodeId ();
mycolor = 42;
mykey = world_myrank;
MPI_Comm_split (MPI_COMM_WORLD, mycolor, mykey, &sub_comm);
MPI_Comm_size(sub_comm, &sub_numtasks);
......
examples/HACC-IO/miniHACC-SoA-MPIIO.cpp
View file @ 1c90dfb0
......@@ -21,6 +21,7 @@ int main (int argc, char * argv[])
double io_bw;
MPI_Comm sub_comm;
MPI_File file_handle;
MPI_Info info;
MPI_Status status;
char output[100];
Tapioca tp;
......@@ -105,10 +106,14 @@ int main (int argc, char * argv[])
mycolor, (double)file_size/(1024*1024), sub_particles);
}
MPI_Info_create ( &info );
MPI_Info_set ( info, "cb_nodes", getenv("TAPIOCA_NBAGGR") );
MPI_Info_set ( info, "bg_nodes_pset", getenv("TAPIOCA_NBAGGR") );
start_time = MPI_Wtime();
MPI_File_open(sub_comm, output,
MPI_MODE_WRONLY, MPI_INFO_NULL, &file_handle);
MPI_MODE_WRONLY, info, &file_handle);
offset = scan_size * sizeof(float);
......@@ -172,7 +177,7 @@ int main (int argc, char * argv[])
start_time = MPI_Wtime();
MPI_File_open(sub_comm, output,
MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handle);
MPI_MODE_RDONLY, info, &file_handle);
if (0 == sub_myrank)
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] Read output file\n", mycolor);
......
examples/HACC-IO/run_check_bgq.sh
View file @ 1c90dfb0
#!/bin/bash
NODES=1024
NODES=128
PPN=16
NPROCS=$((NODES*PPN))
TARGET="/projects/visualization/ftessier/debug"
......@@ -9,7 +9,7 @@ cd $HOME/install/$ARCHI/bin/
export TAPIOCA_DEVNULL=false
export TAPIOCA_COMMSPLIT=true
export TAPIOCA_STRATEGY=TOPOLOGY_AWARE
export TAPIOCA_STRATEGY=UNIFORM
export TAPIOCA_NBAGGR=16
export TAPIOCA_BUFFERSIZE=16777216
export TAPIOCA_PIPELINING=true
......@@ -34,7 +34,7 @@ function updateSettings()
function outputFile ()
{
OUTPUT="$HOME/PUB-IPDPS18/dat/hacc_rw"
OUTPUT="$HOME/xp_tapioca/contention"
OUTPUT="${OUTPUT}/HACC_BGQ_MIRA_${BIN}_${NODES}_${PPN}"
OUTPUT="${OUTPUT}_${TAPIOCA_STRATEGY}_${TAPIOCA_NBAGGR}_${TAPIOCA_BUFFERSIZE}_${PARTICLES}"
}
......@@ -43,13 +43,14 @@ updateSettings
for run in {0..9}
do
for PARTICLES in 5000 15000 25000 35000 50000 100000
for PARTICLES in 15000 25000 35000 50000
do
for BIN in miniHACC-AoS miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO
do
rm $TARGET/*
outputFile
runjob --block $COBALT_PARTNAME --envs $SETTINGS -p $PPN --np $NPROCS : ./$BIN $PARTICLES >> $OUTPUT
runjob --block $COBALT_PARTNAME --envs $SETTINGS -p $PPN --np $NPROCS : ./$BIN $PARTICLES
#>> $OUTPUT
sleep 3
done
done
......
examples/HACC-IO/run_check_xc40.sh
View file @ 1c90dfb0
#!/bin/bash
NODES=256
NODES=1024
PPN=16
NPROCS=$((NODES*PPN))
TARGET="/lus/theta-fs0/projects/Performance/ftessier/HACC"
DDT="/soft/debuggers/forge/bin/ddt --connect"
STRIPE_COUNT=48
STRIPE_SIZE=8388608
AGGR_MULT=1
cd $HOME/install/$ARCHI/bin/
export TAPIOCA_DEVNULL=false
export TAPIOCA_COMMSPLIT=true
export TAPIOCA_STRATEGY=TOPOLOGY_AWARE
export TAPIOCA_NBAGGR=48
export TAPIOCA_NBAGGR=192
export TAPIOCA_BUFFERSIZE=16777216
function setLustreFS ()
......@@ -35,28 +36,34 @@ function updateSettings()
SETTINGS="$SETTINGS -e MPICH_RMA_OVER_DMAPP=1"
SETTINGS="$SETTINGS -e MPICH_MPIIO_AGGREGATOR_PLACEMENT_DISPLAY=1"
SETTINGS="$SETTINGS -e MPICH_MPIIO_HINTS=*:cray_cb_nodes_multiplier=1"
SETTINGS="$SETTINGS -e MPICH_MPIIO_HINTS=*:cray_cb_nodes_multiplier=$AGGR_MULT"
}
function outputFile ()
{
OUTPUT="$HOME/PUB-IPDPS18/dat/hacc_rw"
OUTPUT="$HOME/PUB-IPDPS18/dat/aggr_count"
OUTPUT="${OUTPUT}/HACC_XC40_${BIN}_${NODES}_${PPN}"
OUTPUT="${OUTPUT}_${TAPIOCA_STRATEGY}_${TAPIOCA_NBAGGR}_${TAPIOCA_BUFFERSIZE}_${PARTICLES}"
}
updateSettings
for run in {0..9}
do
for PARTICLES in 5000 15000 25000 35000 50000 100000
for PARTICLES in 25000
do
for BIN in miniHACC-AoS miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO
do
outputFile
setLustreFS >> $OUTPUT
aprun $SETTINGS -n $NPROCS -N $PPN ./$BIN $PARTICLES >> $OUTPUT
sleep 5
for AGGR in 48 96 144 192
do
AGGR_MULT=$((AGGR/STRIPE_COUNT))
echo "cray_cb_nodes_multiplier=$AGGR_MULT"
export TAPIOCA_NBAGGR=$AGGR
updateSettings
for BIN in miniHACC-AoS miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO
do
outputFile
setLustreFS >> $OUTPUT
aprun $SETTINGS -n $NPROCS -N $PPN ./$BIN $PARTICLES >> $OUTPUT
sleep 5
done
done
done
done
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment