Update HACC-IO to use the new TAPIOCA API

examples/HACC-IO/Makefile

 include ../make.inc/Makefile.$(ARCHI)
 
-all: miniHACC-AoS miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO
+all: miniHACC-AoS #miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO
 
 miniHACC-AoS:miniHACC-AoS.cpp
 	$(MPICXX) $(MPI_CFLAGS) $? -o $@ $(MPI_LDFLAGS)

examples/HACC-IO/miniHACC-AoS.cpp

@@ -20,7 +20,6 @@ int main (int argc, char * argv[])
   double start_time, end_time, tot_time, max_time;
   double io_bw;
   MPI_Comm sub_comm;
-  MPI_File file_handle;
   MPI_Status status;
   char output[100];
   Tapioca tp;
@@ -91,12 +90,12 @@ int main (int argc, char * argv[])
   file_size = particle_size * sub_particles;
   tot_size = particle_size * tot_particles;
 
-  if (sub_myrank == 0) {
-    MPI_File_open(MPI_COMM_SELF, output,
-		  MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL, &file_handle);
-    MPI_File_set_size(file_handle, file_size);
-    MPI_File_close (&file_handle);
-  }
+  // if (sub_myrank == 0) {
+  //   MPI_File_open(MPI_COMM_SELF, output,
+  // 		  MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL, &file_handle);
+  //   MPI_File_set_size(file_handle, file_size);
+  //   MPI_File_close (&file_handle);
+  // }
 
   MPI_Exscan (&num_particles, &scan_size, 1, MPI_LONG_LONG, MPI_SUM, sub_comm);
     
@@ -130,44 +129,43 @@ int main (int argc, char * argv[])
     chunkOffset[i] = chunkOffset[i - 1] + chunkCount[i - 1] * chunkSize[i - 1];
   }
   
-  tp.WriteInitialize (output, chunkCount, chunkSize, chunkOffset, 9, hdr, ARRAY_OF_STRUCTURES, sub_comm);
+  tp.Init (chunkCount, chunkSize, chunkOffset, 9, hdr, sub_comm);
+  tp.setAggregationTier (DDR, "");
+  tp.setTargetTier (HDD, file_size, output);
   /*****************/
   
   start_time = MPI_Wtime();
 
-  MPI_File_open(sub_comm, output,
-		MPI_MODE_WRONLY, MPI_INFO_NULL, &file_handle);
-  
   offset = scan_size * particle_size;
 
-  tp.Write (file_handle, offset, xx, num_particles, MPI_FLOAT, &status);
+  tp.Write (offset, xx, num_particles, MPI_FLOAT, &status);
   offset += num_particles * sizeof(float);
 
-  tp.Write (file_handle, offset, yy, num_particles, MPI_FLOAT, &status);
+  tp.Write (offset, yy, num_particles, MPI_FLOAT, &status);
   offset += num_particles * sizeof(float);
 
-  tp.Write (file_handle, offset, zz, num_particles, MPI_FLOAT, &status);
+  tp.Write (offset, zz, num_particles, MPI_FLOAT, &status);
   offset += num_particles * sizeof(float);
 
-  tp.Write (file_handle, offset, vx, num_particles, MPI_FLOAT, &status);
+  tp.Write (offset, vx, num_particles, MPI_FLOAT, &status);
   offset += num_particles * sizeof(float);
 
-  tp.Write (file_handle, offset, vy, num_particles, MPI_FLOAT, &status);
+  tp.Write (offset, vy, num_particles, MPI_FLOAT, &status);
   offset += num_particles * sizeof(float);
 
-  tp.Write (file_handle, offset, vz, num_particles, MPI_FLOAT, &status);
+  tp.Write (offset, vz, num_particles, MPI_FLOAT, &status);
   offset += num_particles * sizeof(float);
 
-  tp.Write (file_handle, offset, phi, num_particles, MPI_FLOAT, &status);
+  tp.Write (offset, phi, num_particles, MPI_FLOAT, &status);
   offset += num_particles * sizeof(float);
 
-  tp.Write (file_handle, offset, pid, num_particles, MPI_LONG_LONG, &status);
+  tp.Write (offset, pid, num_particles, MPI_LONG_LONG, &status);
   offset += num_particles * sizeof(int64_t);
 
-  tp.Write (file_handle, offset, mask, num_particles, MPI_UNSIGNED_SHORT, &status);
-  
-  MPI_File_close (&file_handle);
+  tp.Write (offset, mask, num_particles, MPI_UNSIGNED_SHORT, &status);
 
+  tp.Finalize ();
+  
   end_time = MPI_Wtime();
   tot_time = end_time - start_time;
   MPI_Reduce (&tot_time, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
@@ -180,8 +178,6 @@ int main (int argc, char * argv[])
 
   MPI_Barrier (MPI_COMM_WORLD);
 
-  tp.Finalize ();
-
   /*****************/
   /*     READ      */
   /*****************/
@@ -202,47 +198,46 @@ int main (int argc, char * argv[])
   /*****************/
   /* INIT TAPIOCA  */
   /*****************/
-  tp.ReadInitialize (output, chunkCount, chunkSize, chunkOffset, 9, hdr, ARRAY_OF_STRUCTURES, sub_comm);
+  tp.Init (chunkCount, chunkSize, chunkOffset, 9, hdr, sub_comm);
+  tp.setAggregationTier (DDR, "");
+  tp.setTargetTier (HDD, file_size, output);
   /*****************/
 
   start_time = MPI_Wtime();
 
-  MPI_File_open(sub_comm, output,
-		MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handle);
-
-  if (0 == sub_myrank)
+    if (0 == sub_myrank)
     fprintf (stdout, GREEN "[INFO]" RESET " [%08d] Read output file\n", mycolor);
 
   offset = scan_size * particle_size;
 
-  tp.Read (file_handle, offset, xx_r, num_particles, MPI_FLOAT, &status);
+  tp.Read (offset, xx_r, num_particles, MPI_FLOAT, &status);
   offset += num_particles * sizeof(float);
 
-  tp.Read (file_handle, offset, yy_r, num_particles, MPI_FLOAT, &status);
+  tp.Read (offset, yy_r, num_particles, MPI_FLOAT, &status);
   offset += num_particles * sizeof(float);
 
-  tp.Read (file_handle, offset, zz_r, num_particles, MPI_FLOAT, &status);
+  tp.Read (offset, zz_r, num_particles, MPI_FLOAT, &status);
   offset += num_particles * sizeof(float);
 
-  tp.Read (file_handle, offset, vx_r, num_particles, MPI_FLOAT, &status);
+  tp.Read (offset, vx_r, num_particles, MPI_FLOAT, &status);
   offset += num_particles * sizeof(float);
 
-  tp.Read (file_handle, offset, vy_r, num_particles, MPI_FLOAT, &status);
+  tp.Read (offset, vy_r, num_particles, MPI_FLOAT, &status);
   offset += num_particles * sizeof(float);
 
-  tp.Read (file_handle, offset, vz_r, num_particles, MPI_FLOAT, &status);
+  tp.Read (offset, vz_r, num_particles, MPI_FLOAT, &status);
   offset += num_particles * sizeof(float);
 
-  tp.Read (file_handle, offset, phi_r, num_particles, MPI_FLOAT, &status);
+  tp.Read (offset, phi_r, num_particles, MPI_FLOAT, &status);
   offset += num_particles * sizeof(float);
 
-  tp.Read (file_handle, offset, pid_r, num_particles, MPI_LONG_LONG, &status);
+  tp.Read (offset, pid_r, num_particles, MPI_LONG_LONG, &status);
   offset += num_particles * sizeof(int64_t);
 
-  tp.Read (file_handle, offset, mask_r, num_particles, MPI_UNSIGNED_SHORT, &status);
-  
-  MPI_File_close (&file_handle);
+  tp.Read (offset, mask_r, num_particles, MPI_UNSIGNED_SHORT, &status);
 
+  tp.Finalize ();
+  
   end_time = MPI_Wtime();
   tot_time = end_time - start_time;
   MPI_Reduce (&tot_time, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);

examples/HACC-IO/miniHACC-SoA.cpp

@@ -131,7 +131,7 @@ int main (int argc, char * argv[])
     chunkOffset[i] += scan_size * chunkSize[i];
   }
 
-  tp.WriteInitialize (output, chunkCount, chunkSize, chunkOffset, 9, hdr, STRUCTURE_OF_ARRAYS, sub_comm);
+  tp.Initialize (chunkCount, chunkSize, chunkOffset, 9, hdr, STRUCTURE_OF_ARRAYS, sub_comm);
   /*****************/
 
   start_time = MPI_Wtime();
@@ -203,7 +203,7 @@ int main (int argc, char * argv[])
   /*****************/
   /* INIT TAPIOCA  */
   /*****************/
-  tp.ReadInitialize (output, chunkCount, chunkSize, chunkOffset, 9, hdr, ARRAY_OF_STRUCTURES, sub_comm);
+  tp.Initialize (chunkCount, chunkSize, chunkOffset, 9, hdr, ARRAY_OF_STRUCTURES, sub_comm);
   /*****************/
 
   start_time = MPI_Wtime();

examples/HACC-IO/run_debug_xc40.sh

 #!/bin/bash
 
-NODES=8
+NODES=64
 PPN=16
 NPROCS=$((NODES*PPN))
 TARGET="/lus/theta-fs0/projects/Performance/ftessier/HACC"
 DDT="/soft/debuggers/forge/bin/ddt --connect"
-STRIPE_COUNT=2
+STRIPE_COUNT=8
 STRIPE_SIZE=8388608
 AGGR_MULT=1
 
@@ -14,7 +14,7 @@ cd $HOME/install/$ARCHI/bin/
 export TAPIOCA_DEVNULL=false
 export TAPIOCA_COMMSPLIT=true
 export TAPIOCA_STRATEGY=TOPOLOGY_AWARE
-export TAPIOCA_NBAGGR=2
+export TAPIOCA_NBAGGR=8
 export TAPIOCA_BUFFERSIZE=16777216
 
 function setLustreFS ()
@@ -50,5 +50,5 @@ PARTICLES=25000
 BIN=miniHACC-AoS
 
 updateSettings
-setLustreFS >> $OUTPUT
-$DDT aprun $SETTINGS -n $NPROCS -N $PPN ./$BIN $PARTICLES
+setLustreFS
+aprun $SETTINGS -n $NPROCS -N $PPN ./$BIN $PARTICLES