User-defined number of particles per rank for HACC-IO

examples/HACC-IO/miniHACC-AoS-MPIIO.cpp

@@ -14,7 +14,7 @@
 int main (int argc, char * argv[])
 {
   int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank, file_id;
-  int64_t num_particles = 25000;
+  int64_t num_particles;
   int64_t sub_particles, tot_particles, particle_size, file_size, tot_size;
   int64_t scan_size = 0, offset;
   double start_time, end_time, tot_time, max_time;
@@ -36,6 +36,8 @@ int main (int argc, char * argv[])
   MPI_Comm_size(sub_comm, &sub_numtasks);
   MPI_Comm_rank(sub_comm, &sub_myrank);
 
+  num_particles = atoi ( argv[1] );
+
   file_id = 0;
   if ( argv[1] != NULL )
     file_id = atoi ( argv[1] );

examples/HACC-IO/miniHACC-AoS.cpp

@@ -14,7 +14,7 @@
 int main (int argc, char * argv[])
 {
   int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank, i, file_id;
-  int64_t num_particles = 25000;
+  int64_t num_particles;
   int64_t sub_particles, tot_particles, particle_size, file_size, tot_size;
   int64_t scan_size = 0, offset, hdr = 0;
   double start_time, end_time, tot_time, max_time;
@@ -38,9 +38,11 @@ int main (int argc, char * argv[])
   MPI_Comm_size(sub_comm, &sub_numtasks);
   MPI_Comm_rank(sub_comm, &sub_myrank);
 
+  num_particles = atoi ( argv[1] );
+
   file_id = 0;
-  if ( argv[1] != NULL )
-    file_id = atoi ( argv[1] );
+  if ( argv[2] != NULL )
+    file_id = atoi ( argv[2] );
 		      
 #ifdef BGQ
   snprintf (output, 100, "/projects/visualization/ftessier/debug/HACC-SOA-%08d-%d.dat", mycolor, file_id);

examples/HACC-IO/miniHACC-SoA-MPIIO.cpp

@@ -14,7 +14,7 @@
 int main (int argc, char * argv[])
 {
   int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank, file_id;
-  int64_t num_particles = 25000;
+  int64_t num_particles;
   int64_t sub_particles, tot_particles, particle_size, file_size, tot_size;
   int64_t scan_size = 0, offset;
   double start_time, end_time, tot_time, max_time;
@@ -36,6 +36,8 @@ int main (int argc, char * argv[])
   MPI_Comm_size(sub_comm, &sub_numtasks);
   MPI_Comm_rank(sub_comm, &sub_myrank);
 
+  num_particles = atoi ( argv[1] );
+
   file_id = 0;
   if ( argv[1] != NULL )
     file_id = atoi ( argv[1] );

examples/HACC-IO/miniHACC-SoA.cpp

@@ -14,7 +14,7 @@
 int main (int argc, char * argv[])
 {
   int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank, i, file_id;
-  int64_t num_particles = 25000;
+  int64_t num_particles;
   int64_t sub_particles, tot_particles, particle_size, file_size, tot_size;
   int64_t scan_size = 0, offset, hdr = 0;
   double start_time, end_time, tot_time, max_time;
@@ -38,6 +38,8 @@ int main (int argc, char * argv[])
   MPI_Comm_size(sub_comm, &sub_numtasks);
   MPI_Comm_rank(sub_comm, &sub_myrank);
 
+  num_particles = atoi ( argv[1] );
+
   file_id = 0;
   if ( argv[1] != NULL )
     file_id = atoi ( argv[1] );

examples/HACC-IO/run_check_xc40.sh

 #!/bin/bash
 
-VARS="-e MPICH_RMA_OVER_DMAPP=1 -e MPICH_MPIIO_AGGREGATOR_PLACEMENT_DISPLAY=1 -e MPICH_MPIIO_HINTS=*:cray_cb_nodes_multiplier=8"
-NODES=64
+NODES=1024
 PPN=16
 NPROCS=$((NODES*PPN))
 TARGET="/lus/theta-fs0/projects/Performance/ftessier/HACC"
 DDT="/soft/debuggers/forge/bin/ddt --connect"
+STRIPE_COUNT=48
+STRIPE_SIZE=8388608
 
 cd $HOME/TAPIOCA/examples/HACC-IO
 
 export TAPIOCA_DEVNULL=false
 export TAPIOCA_COMMSPLIT=true
 export TAPIOCA_STRATEGY=TOPOLOGY_AWARE
-export TAPIOCA_NBAGGR=8
+export TAPIOCA_NBAGGR=64
 #export TAPIOCA_BUFFERSIZE=2097152
 #export TAPIOCA_BUFFERSIZE=4194304
 #export TAPIOCA_BUFFERSIZE=8388608
 export TAPIOCA_BUFFERSIZE=16777216
+#export TAPIOCA_BUFFERSIZE=33554432
+
+VARS="-e MPICH_RMA_OVER_DMAPP=1 -e MPICH_MPIIO_AGGREGATOR_PLACEMENT_DISPLAY=1 -e MPICH_MPIIO_HINTS=*:cray_cb_nodes_multiplier=1"
+
+function setLustreFS ()
+{
+    rm $TARGET/*
+    lfs setstripe --stripe-count $STRIPE_COUNT --stripe-size $STRIPE_SIZE $TARGET
+    lfs getstripe $TARGET
+}
 
 function updateSettings()
 {
@@ -30,17 +41,28 @@ function updateSettings()
     SETTINGS="$SETTINGS -e MPICH_RMA_OVER_DMAPP=1"
 }
 
+for PARTICLES in 5000 15000 25000 35000 50000
+do
+    updateSettings
+    setLustreFS
+    
+    rm $TARGET/*
+    aprun $VARS $SETTINGS -n $NPROCS -N $PPN ./miniHACC-AoS $PARTICLES 1
+    sleep 5
+    
+    rm $TARGET/*
+    aprun $VARS $SETTINGS -n $NPROCS -N $PPN ./miniHACC-AoS-MPIIO $PARTICLES 1
+    sleep 5
+    
+    rm $TARGET/*
+    aprun $VARS $SETTINGS -n $NPROCS -N $PPN ./miniHACC-SoA $PARTICLES 1
+    sleep 5
+    
+    rm $TARGET/*
+    aprun $VARS $SETTINGS -n $NPROCS -N $PPN ./miniHACC-SoA-MPIIO $PARTICLES 1
+    sleep 5
+done
 
-rm $TARGET/*
-updateSettings
-
-aprun $VARS $SETTINGS -n $NPROCS -N $PPN ./miniHACC-AoS 1
-sleep 5
-aprun $VARS $SETTINGS -n $NPROCS -N $PPN ./miniHACC-AoS-MPIIO 1
-sleep 5
-aprun $VARS $SETTINGS -n $NPROCS -N $PPN ./miniHACC-SoA 1
-sleep 5
-aprun $VARS $SETTINGS -n $NPROCS -N $PPN ./miniHACC-SoA-MPIIO 1
 #########################
 # Array of Structures
 #########################