Commit dbe6daa4 authored by Francois Tessier's avatar Francois Tessier

In HACC-IO, add MPI file information to set custom values for MPI I/O hints + update run scripts

parent b2e0c318
...@@ -21,6 +21,7 @@ int main (int argc, char * argv[]) ...@@ -21,6 +21,7 @@ int main (int argc, char * argv[])
double io_bw; double io_bw;
MPI_Comm sub_comm; MPI_Comm sub_comm;
MPI_File file_handle; MPI_File file_handle;
MPI_Info info;
MPI_Status status; MPI_Status status;
char output[100]; char output[100];
Tapioca tp; Tapioca tp;
...@@ -105,10 +106,14 @@ int main (int argc, char * argv[]) ...@@ -105,10 +106,14 @@ int main (int argc, char * argv[])
mycolor, (double)file_size/(1024*1024), sub_particles); mycolor, (double)file_size/(1024*1024), sub_particles);
} }
MPI_Info_create ( &info );
MPI_Info_set ( info, "cb_nodes", getenv("TAPIOCA_NBAGGR") );
MPI_Info_set ( info, "bg_nodes_pset", getenv("TAPIOCA_NBAGGR") );
start_time = MPI_Wtime(); start_time = MPI_Wtime();
MPI_File_open(sub_comm, output, MPI_File_open(sub_comm, output,
MPI_MODE_WRONLY, MPI_INFO_NULL, &file_handle); MPI_MODE_WRONLY, info, &file_handle);
offset = scan_size * particle_size; offset = scan_size * particle_size;
...@@ -172,7 +177,7 @@ int main (int argc, char * argv[]) ...@@ -172,7 +177,7 @@ int main (int argc, char * argv[])
start_time = MPI_Wtime(); start_time = MPI_Wtime();
MPI_File_open(sub_comm, output, MPI_File_open(sub_comm, output,
MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handle); MPI_MODE_RDONLY, info, &file_handle);
if (0 == sub_myrank) if (0 == sub_myrank)
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] Read output file\n", mycolor); fprintf (stdout, GREEN "[INFO]" RESET " [%08d] Read output file\n", mycolor);
......
...@@ -30,9 +30,10 @@ int main (int argc, char * argv[]) ...@@ -30,9 +30,10 @@ int main (int argc, char * argv[])
MPI_Init(&argc, &argv); MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &world_numtasks); MPI_Comm_size(MPI_COMM_WORLD, &world_numtasks);
MPI_Comm_rank(MPI_COMM_WORLD, &world_myrank); MPI_Comm_rank(MPI_COMM_WORLD, &world_myrank);
mycolor = tp.topology.BridgeNodeId (); //mycolor = tp.topology.BridgeNodeId ();
mykey = world_myrank; mycolor = 42;
mykey = world_myrank;
MPI_Comm_split (MPI_COMM_WORLD, mycolor, mykey, &sub_comm); MPI_Comm_split (MPI_COMM_WORLD, mycolor, mykey, &sub_comm);
MPI_Comm_size(sub_comm, &sub_numtasks); MPI_Comm_size(sub_comm, &sub_numtasks);
......
...@@ -21,6 +21,7 @@ int main (int argc, char * argv[]) ...@@ -21,6 +21,7 @@ int main (int argc, char * argv[])
double io_bw; double io_bw;
MPI_Comm sub_comm; MPI_Comm sub_comm;
MPI_File file_handle; MPI_File file_handle;
MPI_Info info;
MPI_Status status; MPI_Status status;
char output[100]; char output[100];
Tapioca tp; Tapioca tp;
...@@ -105,10 +106,14 @@ int main (int argc, char * argv[]) ...@@ -105,10 +106,14 @@ int main (int argc, char * argv[])
mycolor, (double)file_size/(1024*1024), sub_particles); mycolor, (double)file_size/(1024*1024), sub_particles);
} }
MPI_Info_create ( &info );
MPI_Info_set ( info, "cb_nodes", getenv("TAPIOCA_NBAGGR") );
MPI_Info_set ( info, "bg_nodes_pset", getenv("TAPIOCA_NBAGGR") );
start_time = MPI_Wtime(); start_time = MPI_Wtime();
MPI_File_open(sub_comm, output, MPI_File_open(sub_comm, output,
MPI_MODE_WRONLY, MPI_INFO_NULL, &file_handle); MPI_MODE_WRONLY, info, &file_handle);
offset = scan_size * sizeof(float); offset = scan_size * sizeof(float);
...@@ -172,7 +177,7 @@ int main (int argc, char * argv[]) ...@@ -172,7 +177,7 @@ int main (int argc, char * argv[])
start_time = MPI_Wtime(); start_time = MPI_Wtime();
MPI_File_open(sub_comm, output, MPI_File_open(sub_comm, output,
MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handle); MPI_MODE_RDONLY, info, &file_handle);
if (0 == sub_myrank) if (0 == sub_myrank)
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] Read output file\n", mycolor); fprintf (stdout, GREEN "[INFO]" RESET " [%08d] Read output file\n", mycolor);
......
#!/bin/bash #!/bin/bash
NODES=1024 NODES=128
PPN=16 PPN=16
NPROCS=$((NODES*PPN)) NPROCS=$((NODES*PPN))
TARGET="/projects/visualization/ftessier/debug" TARGET="/projects/visualization/ftessier/debug"
...@@ -9,7 +9,7 @@ cd $HOME/install/$ARCHI/bin/ ...@@ -9,7 +9,7 @@ cd $HOME/install/$ARCHI/bin/
export TAPIOCA_DEVNULL=false export TAPIOCA_DEVNULL=false
export TAPIOCA_COMMSPLIT=true export TAPIOCA_COMMSPLIT=true
export TAPIOCA_STRATEGY=TOPOLOGY_AWARE export TAPIOCA_STRATEGY=UNIFORM
export TAPIOCA_NBAGGR=16 export TAPIOCA_NBAGGR=16
export TAPIOCA_BUFFERSIZE=16777216 export TAPIOCA_BUFFERSIZE=16777216
export TAPIOCA_PIPELINING=true export TAPIOCA_PIPELINING=true
...@@ -34,7 +34,7 @@ function updateSettings() ...@@ -34,7 +34,7 @@ function updateSettings()
function outputFile () function outputFile ()
{ {
OUTPUT="$HOME/PUB-IPDPS18/dat/hacc_rw" OUTPUT="$HOME/xp_tapioca/contention"
OUTPUT="${OUTPUT}/HACC_BGQ_MIRA_${BIN}_${NODES}_${PPN}" OUTPUT="${OUTPUT}/HACC_BGQ_MIRA_${BIN}_${NODES}_${PPN}"
OUTPUT="${OUTPUT}_${TAPIOCA_STRATEGY}_${TAPIOCA_NBAGGR}_${TAPIOCA_BUFFERSIZE}_${PARTICLES}" OUTPUT="${OUTPUT}_${TAPIOCA_STRATEGY}_${TAPIOCA_NBAGGR}_${TAPIOCA_BUFFERSIZE}_${PARTICLES}"
} }
...@@ -43,13 +43,14 @@ updateSettings ...@@ -43,13 +43,14 @@ updateSettings
for run in {0..9} for run in {0..9}
do do
for PARTICLES in 5000 15000 25000 35000 50000 100000 for PARTICLES in 15000 25000 35000 50000
do do
for BIN in miniHACC-AoS miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO for BIN in miniHACC-AoS miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO
do do
rm $TARGET/* rm $TARGET/*
outputFile outputFile
runjob --block $COBALT_PARTNAME --envs $SETTINGS -p $PPN --np $NPROCS : ./$BIN $PARTICLES >> $OUTPUT runjob --block $COBALT_PARTNAME --envs $SETTINGS -p $PPN --np $NPROCS : ./$BIN $PARTICLES
#>> $OUTPUT
sleep 3 sleep 3
done done
done done
......
...@@ -7,13 +7,14 @@ TARGET="/lus/theta-fs0/projects/Performance/ftessier/HACC" ...@@ -7,13 +7,14 @@ TARGET="/lus/theta-fs0/projects/Performance/ftessier/HACC"
DDT="/soft/debuggers/forge/bin/ddt --connect" DDT="/soft/debuggers/forge/bin/ddt --connect"
STRIPE_COUNT=48 STRIPE_COUNT=48
STRIPE_SIZE=8388608 STRIPE_SIZE=8388608
AGGR_MULT=1
cd $HOME/install/$ARCHI/bin/ cd $HOME/install/$ARCHI/bin/
export TAPIOCA_DEVNULL=false export TAPIOCA_DEVNULL=false
export TAPIOCA_COMMSPLIT=true export TAPIOCA_COMMSPLIT=true
export TAPIOCA_STRATEGY=TOPOLOGY_AWARE export TAPIOCA_STRATEGY=TOPOLOGY_AWARE
export TAPIOCA_NBAGGR=1 export TAPIOCA_NBAGGR=192
export TAPIOCA_BUFFERSIZE=16777216 export TAPIOCA_BUFFERSIZE=16777216
function setLustreFS () function setLustreFS ()
...@@ -35,28 +36,34 @@ function updateSettings() ...@@ -35,28 +36,34 @@ function updateSettings()
SETTINGS="$SETTINGS -e MPICH_RMA_OVER_DMAPP=1" SETTINGS="$SETTINGS -e MPICH_RMA_OVER_DMAPP=1"
SETTINGS="$SETTINGS -e MPICH_MPIIO_AGGREGATOR_PLACEMENT_DISPLAY=1" SETTINGS="$SETTINGS -e MPICH_MPIIO_AGGREGATOR_PLACEMENT_DISPLAY=1"
SETTINGS="$SETTINGS -e MPICH_MPIIO_HINTS=*:cray_cb_nodes_multiplier=3" SETTINGS="$SETTINGS -e MPICH_MPIIO_HINTS=*:cray_cb_nodes_multiplier=$AGGR_MULT"
} }
function outputFile () function outputFile ()
{ {
OUTPUT="$HOME/PUB-IPDPS18/dat/hacc_rw" OUTPUT="$HOME/PUB-IPDPS18/dat/aggr_count"
OUTPUT="${OUTPUT}/HACC_XC40_FP8N_${BIN}_${NODES}_${PPN}" OUTPUT="${OUTPUT}/HACC_XC40_${BIN}_${NODES}_${PPN}"
OUTPUT="${OUTPUT}_${TAPIOCA_STRATEGY}_${TAPIOCA_NBAGGR}_${TAPIOCA_BUFFERSIZE}_${PARTICLES}" OUTPUT="${OUTPUT}_${TAPIOCA_STRATEGY}_${TAPIOCA_NBAGGR}_${TAPIOCA_BUFFERSIZE}_${PARTICLES}"
} }
updateSettings
for run in {0..9} for run in {0..9}
do do
for PARTICLES in 5000 15000 25000 35000 50000 100000 for PARTICLES in 25000
do do
for BIN in miniHACC-AoS miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO for AGGR in 48 96 144 192
do do
outputFile AGGR_MULT=$((AGGR/STRIPE_COUNT))
setLustreFS >> $OUTPUT echo "cray_cb_nodes_multiplier=$AGGR_MULT"
aprun $SETTINGS -n $NPROCS -N $PPN ./$BIN $PARTICLES >> $OUTPUT export TAPIOCA_NBAGGR=$AGGR
sleep 5 updateSettings
for BIN in miniHACC-AoS miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO
do
outputFile
setLustreFS >> $OUTPUT
aprun $SETTINGS -n $NPROCS -N $PPN ./$BIN $PARTICLES >> $OUTPUT
sleep 5
done
done done
done done
done done
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment