Commit 109885b6 authored by Francois Tessier's avatar Francois Tessier

User-defined number of particles per rank for HACC-IO

parent 8bab99f2
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
int main (int argc, char * argv[]) int main (int argc, char * argv[])
{ {
int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank, file_id; int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank, file_id;
int64_t num_particles = 25000; int64_t num_particles;
int64_t sub_particles, tot_particles, particle_size, file_size, tot_size; int64_t sub_particles, tot_particles, particle_size, file_size, tot_size;
int64_t scan_size = 0, offset; int64_t scan_size = 0, offset;
double start_time, end_time, tot_time, max_time; double start_time, end_time, tot_time, max_time;
...@@ -36,6 +36,8 @@ int main (int argc, char * argv[]) ...@@ -36,6 +36,8 @@ int main (int argc, char * argv[])
MPI_Comm_size(sub_comm, &sub_numtasks); MPI_Comm_size(sub_comm, &sub_numtasks);
MPI_Comm_rank(sub_comm, &sub_myrank); MPI_Comm_rank(sub_comm, &sub_myrank);
num_particles = atoi ( argv[1] );
file_id = 0; file_id = 0;
if ( argv[1] != NULL ) if ( argv[1] != NULL )
file_id = atoi ( argv[1] ); file_id = atoi ( argv[1] );
......
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
int main (int argc, char * argv[]) int main (int argc, char * argv[])
{ {
int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank, i, file_id; int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank, i, file_id;
int64_t num_particles = 25000; int64_t num_particles;
int64_t sub_particles, tot_particles, particle_size, file_size, tot_size; int64_t sub_particles, tot_particles, particle_size, file_size, tot_size;
int64_t scan_size = 0, offset, hdr = 0; int64_t scan_size = 0, offset, hdr = 0;
double start_time, end_time, tot_time, max_time; double start_time, end_time, tot_time, max_time;
...@@ -38,9 +38,11 @@ int main (int argc, char * argv[]) ...@@ -38,9 +38,11 @@ int main (int argc, char * argv[])
MPI_Comm_size(sub_comm, &sub_numtasks); MPI_Comm_size(sub_comm, &sub_numtasks);
MPI_Comm_rank(sub_comm, &sub_myrank); MPI_Comm_rank(sub_comm, &sub_myrank);
num_particles = atoi ( argv[1] );
file_id = 0; file_id = 0;
if ( argv[1] != NULL ) if ( argv[2] != NULL )
file_id = atoi ( argv[1] ); file_id = atoi ( argv[2] );
#ifdef BGQ #ifdef BGQ
snprintf (output, 100, "/projects/visualization/ftessier/debug/HACC-SOA-%08d-%d.dat", mycolor, file_id); snprintf (output, 100, "/projects/visualization/ftessier/debug/HACC-SOA-%08d-%d.dat", mycolor, file_id);
......
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
int main (int argc, char * argv[]) int main (int argc, char * argv[])
{ {
int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank, file_id; int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank, file_id;
int64_t num_particles = 25000; int64_t num_particles;
int64_t sub_particles, tot_particles, particle_size, file_size, tot_size; int64_t sub_particles, tot_particles, particle_size, file_size, tot_size;
int64_t scan_size = 0, offset; int64_t scan_size = 0, offset;
double start_time, end_time, tot_time, max_time; double start_time, end_time, tot_time, max_time;
...@@ -36,6 +36,8 @@ int main (int argc, char * argv[]) ...@@ -36,6 +36,8 @@ int main (int argc, char * argv[])
MPI_Comm_size(sub_comm, &sub_numtasks); MPI_Comm_size(sub_comm, &sub_numtasks);
MPI_Comm_rank(sub_comm, &sub_myrank); MPI_Comm_rank(sub_comm, &sub_myrank);
num_particles = atoi ( argv[1] );
file_id = 0; file_id = 0;
if ( argv[1] != NULL ) if ( argv[1] != NULL )
file_id = atoi ( argv[1] ); file_id = atoi ( argv[1] );
......
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
int main (int argc, char * argv[]) int main (int argc, char * argv[])
{ {
int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank, i, file_id; int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank, i, file_id;
int64_t num_particles = 25000; int64_t num_particles;
int64_t sub_particles, tot_particles, particle_size, file_size, tot_size; int64_t sub_particles, tot_particles, particle_size, file_size, tot_size;
int64_t scan_size = 0, offset, hdr = 0; int64_t scan_size = 0, offset, hdr = 0;
double start_time, end_time, tot_time, max_time; double start_time, end_time, tot_time, max_time;
...@@ -38,6 +38,8 @@ int main (int argc, char * argv[]) ...@@ -38,6 +38,8 @@ int main (int argc, char * argv[])
MPI_Comm_size(sub_comm, &sub_numtasks); MPI_Comm_size(sub_comm, &sub_numtasks);
MPI_Comm_rank(sub_comm, &sub_myrank); MPI_Comm_rank(sub_comm, &sub_myrank);
num_particles = atoi ( argv[1] );
file_id = 0; file_id = 0;
if ( argv[1] != NULL ) if ( argv[1] != NULL )
file_id = atoi ( argv[1] ); file_id = atoi ( argv[1] );
......
#!/bin/bash #!/bin/bash
VARS="-e MPICH_RMA_OVER_DMAPP=1 -e MPICH_MPIIO_AGGREGATOR_PLACEMENT_DISPLAY=1 -e MPICH_MPIIO_HINTS=*:cray_cb_nodes_multiplier=8" NODES=1024
NODES=64
PPN=16 PPN=16
NPROCS=$((NODES*PPN)) NPROCS=$((NODES*PPN))
TARGET="/lus/theta-fs0/projects/Performance/ftessier/HACC" TARGET="/lus/theta-fs0/projects/Performance/ftessier/HACC"
DDT="/soft/debuggers/forge/bin/ddt --connect" DDT="/soft/debuggers/forge/bin/ddt --connect"
STRIPE_COUNT=48
STRIPE_SIZE=8388608
cd $HOME/TAPIOCA/examples/HACC-IO cd $HOME/TAPIOCA/examples/HACC-IO
export TAPIOCA_DEVNULL=false export TAPIOCA_DEVNULL=false
export TAPIOCA_COMMSPLIT=true export TAPIOCA_COMMSPLIT=true
export TAPIOCA_STRATEGY=TOPOLOGY_AWARE export TAPIOCA_STRATEGY=TOPOLOGY_AWARE
export TAPIOCA_NBAGGR=8 export TAPIOCA_NBAGGR=64
#export TAPIOCA_BUFFERSIZE=2097152 #export TAPIOCA_BUFFERSIZE=2097152
#export TAPIOCA_BUFFERSIZE=4194304 #export TAPIOCA_BUFFERSIZE=4194304
#export TAPIOCA_BUFFERSIZE=8388608 #export TAPIOCA_BUFFERSIZE=8388608
export TAPIOCA_BUFFERSIZE=16777216 export TAPIOCA_BUFFERSIZE=16777216
#export TAPIOCA_BUFFERSIZE=33554432
VARS="-e MPICH_RMA_OVER_DMAPP=1 -e MPICH_MPIIO_AGGREGATOR_PLACEMENT_DISPLAY=1 -e MPICH_MPIIO_HINTS=*:cray_cb_nodes_multiplier=1"
function setLustreFS ()
{
rm $TARGET/*
lfs setstripe --stripe-count $STRIPE_COUNT --stripe-size $STRIPE_SIZE $TARGET
lfs getstripe $TARGET
}
function updateSettings() function updateSettings()
{ {
...@@ -30,17 +41,28 @@ function updateSettings() ...@@ -30,17 +41,28 @@ function updateSettings()
SETTINGS="$SETTINGS -e MPICH_RMA_OVER_DMAPP=1" SETTINGS="$SETTINGS -e MPICH_RMA_OVER_DMAPP=1"
} }
for PARTICLES in 5000 15000 25000 35000 50000
do
updateSettings
setLustreFS
rm $TARGET/*
aprun $VARS $SETTINGS -n $NPROCS -N $PPN ./miniHACC-AoS $PARTICLES 1
sleep 5
rm $TARGET/*
aprun $VARS $SETTINGS -n $NPROCS -N $PPN ./miniHACC-AoS-MPIIO $PARTICLES 1
sleep 5
rm $TARGET/*
aprun $VARS $SETTINGS -n $NPROCS -N $PPN ./miniHACC-SoA $PARTICLES 1
sleep 5
rm $TARGET/*
aprun $VARS $SETTINGS -n $NPROCS -N $PPN ./miniHACC-SoA-MPIIO $PARTICLES 1
sleep 5
done
rm $TARGET/*
updateSettings
aprun $VARS $SETTINGS -n $NPROCS -N $PPN ./miniHACC-AoS 1
sleep 5
aprun $VARS $SETTINGS -n $NPROCS -N $PPN ./miniHACC-AoS-MPIIO 1
sleep 5
aprun $VARS $SETTINGS -n $NPROCS -N $PPN ./miniHACC-SoA 1
sleep 5
aprun $VARS $SETTINGS -n $NPROCS -N $PPN ./miniHACC-SoA-MPIIO 1
######################### #########################
# Array of Structures # Array of Structures
######################### #########################
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment