In HACC-IO, add MPI file information to set custom values for MPI I/O hints + update run scripts

examples/HACC-IO/miniHACC-AoS-MPIIO.cpp

@@ -21,6 +21,7 @@ int main (int argc, char * argv[])
   double io_bw;
   MPI_Comm sub_comm;
   MPI_File file_handle;
+  MPI_Info info;
   MPI_Status status;
   char output[100];
   Tapioca tp;
@@ -105,10 +106,14 @@ int main (int argc, char * argv[])
 	     mycolor, (double)file_size/(1024*1024), sub_particles);
   }
 
+  MPI_Info_create ( &info );
+  MPI_Info_set ( info, "cb_nodes", getenv("TAPIOCA_NBAGGR") );
+  MPI_Info_set ( info, "bg_nodes_pset", getenv("TAPIOCA_NBAGGR") );
+
   start_time = MPI_Wtime();
 
   MPI_File_open(sub_comm, output,
-		MPI_MODE_WRONLY, MPI_INFO_NULL, &file_handle);
+		MPI_MODE_WRONLY, info, &file_handle);
   
   offset = scan_size * particle_size;
 
@@ -172,7 +177,7 @@ int main (int argc, char * argv[])
   start_time = MPI_Wtime();
 
   MPI_File_open(sub_comm, output,
-		MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handle);
+		MPI_MODE_RDONLY, info, &file_handle);
 
   if (0 == sub_myrank)
     fprintf (stdout, GREEN "[INFO]" RESET " [%08d] Read output file\n", mycolor);

examples/HACC-IO/miniHACC-AoS.cpp

@@ -30,9 +30,10 @@ int main (int argc, char * argv[])
   MPI_Init(&argc, &argv);
   MPI_Comm_size(MPI_COMM_WORLD, &world_numtasks);
   MPI_Comm_rank(MPI_COMM_WORLD, &world_myrank);
-
-  mycolor = tp.topology.BridgeNodeId ();
-    mykey = world_myrank;
+  
+  //mycolor = tp.topology.BridgeNodeId ();
+  mycolor = 42;
+  mykey = world_myrank;
 
   MPI_Comm_split (MPI_COMM_WORLD, mycolor, mykey, &sub_comm);
   MPI_Comm_size(sub_comm, &sub_numtasks);

examples/HACC-IO/miniHACC-SoA-MPIIO.cpp

@@ -21,6 +21,7 @@ int main (int argc, char * argv[])
   double io_bw;
   MPI_Comm sub_comm;
   MPI_File file_handle;
+  MPI_Info info;
   MPI_Status status;
   char output[100];
   Tapioca tp;
@@ -105,10 +106,14 @@ int main (int argc, char * argv[])
 	     mycolor, (double)file_size/(1024*1024), sub_particles);
   }
 
+  MPI_Info_create ( &info );
+  MPI_Info_set ( info, "cb_nodes", getenv("TAPIOCA_NBAGGR") );
+  MPI_Info_set ( info, "bg_nodes_pset", getenv("TAPIOCA_NBAGGR") );
+
   start_time = MPI_Wtime();
 
   MPI_File_open(sub_comm, output,
-		MPI_MODE_WRONLY, MPI_INFO_NULL, &file_handle);
+		MPI_MODE_WRONLY, info, &file_handle);
   
   offset = scan_size * sizeof(float);
 
@@ -172,7 +177,7 @@ int main (int argc, char * argv[])
   start_time = MPI_Wtime();
 
   MPI_File_open(sub_comm, output,
-		MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handle);
+		MPI_MODE_RDONLY, info, &file_handle);
 
   if (0 == sub_myrank)
     fprintf (stdout, GREEN "[INFO]" RESET " [%08d] Read output file\n", mycolor);

examples/HACC-IO/run_check_bgq.sh

 #!/bin/bash
 
-NODES=1024
+NODES=128
 PPN=16
 NPROCS=$((NODES*PPN))
 TARGET="/projects/visualization/ftessier/debug"
@@ -9,7 +9,7 @@ cd $HOME/install/$ARCHI/bin/
 
 export TAPIOCA_DEVNULL=false
 export TAPIOCA_COMMSPLIT=true
-export TAPIOCA_STRATEGY=TOPOLOGY_AWARE
+export TAPIOCA_STRATEGY=UNIFORM
 export TAPIOCA_NBAGGR=16
 export TAPIOCA_BUFFERSIZE=16777216
 export TAPIOCA_PIPELINING=true
@@ -34,7 +34,7 @@ function updateSettings()
 
 function outputFile ()
 {
-    OUTPUT="$HOME/PUB-IPDPS18/dat/hacc_rw"
+    OUTPUT="$HOME/xp_tapioca/contention"
     OUTPUT="${OUTPUT}/HACC_BGQ_MIRA_${BIN}_${NODES}_${PPN}"
     OUTPUT="${OUTPUT}_${TAPIOCA_STRATEGY}_${TAPIOCA_NBAGGR}_${TAPIOCA_BUFFERSIZE}_${PARTICLES}"
 }
@@ -43,13 +43,14 @@ updateSettings
 
 for run in {0..9}
 do
-    for PARTICLES in 5000 15000 25000 35000 50000 100000
+    for PARTICLES in 15000 25000 35000 50000
     do
         for BIN in miniHACC-AoS miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO
         do
 	    rm $TARGET/*
 	    outputFile
-	    runjob --block $COBALT_PARTNAME --envs $SETTINGS -p $PPN --np $NPROCS : ./$BIN $PARTICLES >> $OUTPUT
+	    runjob --block $COBALT_PARTNAME --envs $SETTINGS -p $PPN --np $NPROCS : ./$BIN $PARTICLES 
+	    #>> $OUTPUT
 	    sleep 3
 	done
     done

examples/HACC-IO/run_check_xc40.sh

@@ -7,13 +7,14 @@ TARGET="/lus/theta-fs0/projects/Performance/ftessier/HACC"
 DDT="/soft/debuggers/forge/bin/ddt --connect"
 STRIPE_COUNT=48
 STRIPE_SIZE=8388608
+AGGR_MULT=1
 
 cd $HOME/install/$ARCHI/bin/
 
 export TAPIOCA_DEVNULL=false
 export TAPIOCA_COMMSPLIT=true
 export TAPIOCA_STRATEGY=TOPOLOGY_AWARE
-export TAPIOCA_NBAGGR=1
+export TAPIOCA_NBAGGR=192
 export TAPIOCA_BUFFERSIZE=16777216
 
 function setLustreFS ()
@@ -35,28 +36,34 @@ function updateSettings()
     SETTINGS="$SETTINGS -e MPICH_RMA_OVER_DMAPP=1"
     
     SETTINGS="$SETTINGS -e MPICH_MPIIO_AGGREGATOR_PLACEMENT_DISPLAY=1"
-    SETTINGS="$SETTINGS -e MPICH_MPIIO_HINTS=*:cray_cb_nodes_multiplier=3"
+    SETTINGS="$SETTINGS -e MPICH_MPIIO_HINTS=*:cray_cb_nodes_multiplier=$AGGR_MULT"
 }
 
 function outputFile ()
 {
-    OUTPUT="$HOME/PUB-IPDPS18/dat/hacc_rw"
-    OUTPUT="${OUTPUT}/HACC_XC40_FP8N_${BIN}_${NODES}_${PPN}"
+    OUTPUT="$HOME/PUB-IPDPS18/dat/aggr_count"
+    OUTPUT="${OUTPUT}/HACC_XC40_${BIN}_${NODES}_${PPN}"
     OUTPUT="${OUTPUT}_${TAPIOCA_STRATEGY}_${TAPIOCA_NBAGGR}_${TAPIOCA_BUFFERSIZE}_${PARTICLES}"
 }
 
-updateSettings
-
 for run in {0..9}
 do
-    for PARTICLES in 5000 15000 25000 35000 50000 100000
+    for PARTICLES in 25000
     do
-	for BIN in miniHACC-AoS miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO
-        do
-	    outputFile
-	    setLustreFS >> $OUTPUT
-	    aprun $SETTINGS -n $NPROCS -N $PPN ./$BIN $PARTICLES >> $OUTPUT
-	    sleep 5
+	for AGGR in 48 96 144 192
+	do
+	    AGGR_MULT=$((AGGR/STRIPE_COUNT))
+	    echo "cray_cb_nodes_multiplier=$AGGR_MULT"
+	    export TAPIOCA_NBAGGR=$AGGR
+	    updateSettings
+
+	    for BIN in miniHACC-AoS miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO
+            do
+		outputFile
+		setLustreFS >> $OUTPUT
+		aprun $SETTINGS -n $NPROCS -N $PPN ./$BIN $PARTICLES >> $OUTPUT
+		sleep 5
+	    done
 	done
     done
 done