Skip to content

GitLab

Commit ef46bc18 authored by Francois Tessier's avatar Francois Tessier
Browse files

Global declaration of tiers of memory for aggregation and output file in HACC-IO

parent ff95772e
Hide whitespace changes
Inline Side-by-side
Showing with 64 additions and 36 deletions
examples/HACC-IO/miniHACC-AoS-R.cpp
View file @ ef46bc18
...@@ -11,6 +11,9 @@ ...@@ -11,6 +11,9 @@
#define BLUE "\x1b[34m" #define BLUE "\x1b[34m"
#define RESET "\x1b[0m" #define RESET "\x1b[0m"
mem_t memAggr = NVR;
mem_t memTarg = HDD;
int main (int argc, char * argv[]) int main (int argc, char * argv[])
{ {
int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank, i, file_id; int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank, i, file_id;
...@@ -29,9 +32,8 @@ int main (int argc, char * argv[]) ...@@ -29,9 +32,8 @@ int main (int argc, char * argv[])
MPI_Init(&argc, &argv); MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &world_numtasks); MPI_Comm_size(MPI_COMM_WORLD, &world_numtasks);
MPI_Comm_rank(MPI_COMM_WORLD, &world_myrank); MPI_Comm_rank(MPI_COMM_WORLD, &world_myrank);
mycolor = tp.topology.BridgeNodeId (); mycolor = tp.topology.BridgeNodeId ();
//mycolor = 42;
mykey = world_myrank; mykey = world_myrank;
MPI_Comm_split (MPI_COMM_WORLD, mycolor, mykey, &sub_comm); MPI_Comm_split (MPI_COMM_WORLD, mycolor, mykey, &sub_comm);
...@@ -43,7 +45,7 @@ int main (int argc, char * argv[]) ...@@ -43,7 +45,7 @@ int main (int argc, char * argv[])
file_id = 0; file_id = 0;
if ( argv[2] != NULL ) if ( argv[2] != NULL )
file_id = atoi ( argv[2] ); file_id = atoi ( argv[2] );
#ifdef BGQ #ifdef BGQ
snprintf (output, 100, "/projects/visualization/ftessier/debug/HACC-AOS-%08d-%d.dat", mycolor, file_id); snprintf (output, 100, "/projects/visualization/ftessier/debug/HACC-AOS-%08d-%d.dat", mycolor, file_id);
#elif XC40 #elif XC40
...@@ -140,8 +142,8 @@ int main (int argc, char * argv[]) ...@@ -140,8 +142,8 @@ int main (int argc, char * argv[])
} }
tp.Init (chunkCount, chunkSize, chunkOffset, 9, hdr, sub_comm); tp.Init (chunkCount, chunkSize, chunkOffset, 9, hdr, sub_comm);
tp.setAggregationTier (2, DDR, "/scratch/tmp"); tp.setAggregationTier (2, memAggr, "/scratch/tmp");
tp.setTargetTier (HDD, file_size, output); tp.setTargetTier (memTarg, file_size, output);
/*****************/ /*****************/
start_time = MPI_Wtime(); start_time = MPI_Wtime();
...@@ -197,7 +199,8 @@ int main (int argc, char * argv[]) ...@@ -197,7 +199,8 @@ int main (int argc, char * argv[])
|| (vx[i] != vx_r[i]) || (vy[i] != vy_r[i]) || (vz[i] != vz_r[i]) || (vx[i] != vx_r[i]) || (vy[i] != vy_r[i]) || (vz[i] != vz_r[i])
|| (phi[i] != phi_r[i])|| (pid[i] != pid_r[i]) || (mask[i] != mask_r[i])) || (phi[i] != phi_r[i])|| (pid[i] != pid_r[i]) || (mask[i] != mask_r[i]))
{ {
fprintf (stdout, RED "[ERROR]" RESET " Wrong value for particle %d\n", i); fprintf (stdout, RED "[ERROR]" RESET "[%02d] Wrong value for particle %d (%.2f, %.2f, %.2f, %.2f, %.2f, %.2f, %.2f, %lld, %u )\n",
world_myrank, i, xx_r[i], yy_r[i], zz_r[i], vx_r[i], vy_r[i], vz_r[i], phi_r[i], pid_r[i], mask_r[i] );
MPI_Abort (MPI_COMM_WORLD, -1); MPI_Abort (MPI_COMM_WORLD, -1);
} }
} }
......
examples/HACC-IO/miniHACC-AoS-W.cpp
View file @ ef46bc18
...@@ -45,7 +45,7 @@ int main (int argc, char * argv[]) ...@@ -45,7 +45,7 @@ int main (int argc, char * argv[])
file_id = atoi ( argv[2] ); file_id = atoi ( argv[2] );
#ifdef BGQ #ifdef BGQ
snprintf (output, 100, "/projects/visualization/ftessier/debug/HACC-SOA-%08d-%d.dat", mycolor, file_id); snprintf (output, 100, "/projects/visualization/ftessier/debug/HACC-AOS-%08d-%d.dat", mycolor, file_id);
#elif XC40 #elif XC40
snprintf (output, 100, "/lus/theta-fs0/projects/Performance/ftessier/HACC/HACC-AOS-%08d-%d.dat", mycolor, file_id); snprintf (output, 100, "/lus/theta-fs0/projects/Performance/ftessier/HACC/HACC-AOS-%08d-%d.dat", mycolor, file_id);
#else #else
......
examples/HACC-IO/miniHACC-AoS.cpp
View file @ ef46bc18
...@@ -11,6 +11,9 @@ ...@@ -11,6 +11,9 @@
#define BLUE "\x1b[34m" #define BLUE "\x1b[34m"
#define RESET "\x1b[0m" #define RESET "\x1b[0m"
mem_t memAggr = NVR;
mem_t memTarg = HDD;
int main (int argc, char * argv[]) int main (int argc, char * argv[])
{ {
int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank, i, file_id; int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank, i, file_id;
...@@ -22,7 +25,7 @@ int main (int argc, char * argv[]) ...@@ -22,7 +25,7 @@ int main (int argc, char * argv[])
MPI_Comm sub_comm; MPI_Comm sub_comm;
MPI_Status status; MPI_Status status;
char output[100]; char output[100];
Tapioca tp; Tapioca tp, tp2;
int64_t chunkCount[9], chunkOffset[9]; int64_t chunkCount[9], chunkOffset[9];
int chunkSize[9]; int chunkSize[9];
...@@ -122,8 +125,8 @@ int main (int argc, char * argv[]) ...@@ -122,8 +125,8 @@ int main (int argc, char * argv[])
} }
tp.Init (chunkCount, chunkSize, chunkOffset, 9, hdr, sub_comm); tp.Init (chunkCount, chunkSize, chunkOffset, 9, hdr, sub_comm);
tp.setAggregationTier (5, NVR, "/scratch/tmp"); tp.setAggregationTier (2, memAggr, "/scratch/tmp-write");
tp.setTargetTier (HDD, file_size, output); tp.setTargetTier (memTarg, file_size, output);
/*****************/ /*****************/
start_time = MPI_Wtime(); start_time = MPI_Wtime();
...@@ -190,9 +193,9 @@ int main (int argc, char * argv[]) ...@@ -190,9 +193,9 @@ int main (int argc, char * argv[])
/*****************/ /*****************/
/* INIT TAPIOCA */ /* INIT TAPIOCA */
/*****************/ /*****************/
tp.Init (chunkCount, chunkSize, chunkOffset, 9, hdr, sub_comm); tp2.Init (chunkCount, chunkSize, chunkOffset, 9, hdr, sub_comm);
tp.setAggregationTier (4, NVR, "/scratch/tmp"); tp2.setAggregationTier (2, memAggr, "/scratch/tmp-read");
tp.setTargetTier (HDD, file_size, output); tp2.setTargetTier (memTarg, file_size, output);
/*****************/ /*****************/
start_time = MPI_Wtime(); start_time = MPI_Wtime();
...@@ -202,31 +205,31 @@ int main (int argc, char * argv[]) ...@@ -202,31 +205,31 @@ int main (int argc, char * argv[])
offset = scan_size * particle_size; offset = scan_size * particle_size;
tp.Read (offset, xx_r, num_particles, MPI_FLOAT, &status); tp2.Read (offset, xx_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float); offset += num_particles * sizeof(float);
tp.Read (offset, yy_r, num_particles, MPI_FLOAT, &status); tp2.Read (offset, yy_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float); offset += num_particles * sizeof(float);
tp.Read (offset, zz_r, num_particles, MPI_FLOAT, &status); tp2.Read (offset, zz_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float); offset += num_particles * sizeof(float);
tp.Read (offset, vx_r, num_particles, MPI_FLOAT, &status); tp2.Read (offset, vx_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float); offset += num_particles * sizeof(float);
tp.Read (offset, vy_r, num_particles, MPI_FLOAT, &status); tp2.Read (offset, vy_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float); offset += num_particles * sizeof(float);
tp.Read (offset, vz_r, num_particles, MPI_FLOAT, &status); tp2.Read (offset, vz_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float); offset += num_particles * sizeof(float);
tp.Read (offset, phi_r, num_particles, MPI_FLOAT, &status); tp2.Read (offset, phi_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float); offset += num_particles * sizeof(float);
tp.Read (offset, pid_r, num_particles, MPI_LONG_LONG, &status); tp2.Read (offset, pid_r, num_particles, MPI_LONG_LONG, &status);
offset += num_particles * sizeof(int64_t); offset += num_particles * sizeof(int64_t);
tp.Read (offset, mask_r, num_particles, MPI_UNSIGNED_SHORT, &status); tp2.Read (offset, mask_r, num_particles, MPI_UNSIGNED_SHORT, &status);
end_time = MPI_Wtime(); end_time = MPI_Wtime();
tot_time = end_time - start_time; tot_time = end_time - start_time;
...@@ -238,7 +241,7 @@ int main (int argc, char * argv[]) ...@@ -238,7 +241,7 @@ int main (int argc, char * argv[])
io_bw, (double)tot_size/(1024*1024), max_time * 1000); io_bw, (double)tot_size/(1024*1024), max_time * 1000);
} }
tp.Finalize (); tp2.Finalize ();
/*****************/ /*****************/
/* VERIFICATION */ /* VERIFICATION */
...@@ -248,7 +251,8 @@ int main (int argc, char * argv[]) ...@@ -248,7 +251,8 @@ int main (int argc, char * argv[])
|| (vx[i] != vx_r[i]) || (vy[i] != vy_r[i]) || (vz[i] != vz_r[i]) || (vx[i] != vx_r[i]) || (vy[i] != vy_r[i]) || (vz[i] != vz_r[i])
|| (phi[i] != phi_r[i])|| (pid[i] != pid_r[i]) || (mask[i] != mask_r[i])) || (phi[i] != phi_r[i])|| (pid[i] != pid_r[i]) || (mask[i] != mask_r[i]))
{ {
fprintf (stdout, RED "[ERROR]" RESET "[%03d] Wrong value for particle %d\n", world_myrank, i); fprintf (stdout, RED "[ERROR]" RESET "[%02d] Wrong value for particle %d (%.2f, %.2f, %.2f, %.2f, %.2f, %.2f, %.2f, %lld, %u )\n",
world_myrank, i, xx_r[i], yy_r[i], zz_r[i], vx_r[i], vy_r[i], vz_r[i], phi_r[i], pid_r[i], mask_r[i] );
MPI_Abort (MPI_COMM_WORLD, -1); MPI_Abort (MPI_COMM_WORLD, -1);
} }
} }
......
examples/HACC-IO/run_check_haswell.sh
View file @ ef46bc18
...@@ -13,6 +13,7 @@ export TAPIOCA_COMMSPLIT=true ...@@ -13,6 +13,7 @@ export TAPIOCA_COMMSPLIT=true
export TAPIOCA_STRATEGY=TOPOLOGY_AWARE export TAPIOCA_STRATEGY=TOPOLOGY_AWARE
export TAPIOCA_NBAGGR=4 export TAPIOCA_NBAGGR=4
export TAPIOCA_BUFFERSIZE=16777216 export TAPIOCA_BUFFERSIZE=16777216
#export TAPIOCA_BUFFERSIZE=33554432
export TAPIOCA_PIPELINING=true export TAPIOCA_PIPELINING=true
export PAMID_VERBOSE=1 export PAMID_VERBOSE=1
...@@ -28,28 +29,48 @@ function outputFile () ...@@ -28,28 +29,48 @@ function outputFile ()
printenv | egrep "TAPIOCA_" printenv | egrep "TAPIOCA_"
PARTICLES=200000
#rm $TARGET/* #rm $TARGET/*
#mpirun -f $COBALT_NODEFILE -n $NPROCS ./miniHACC-AoS 15000 #mpirun -f $COBALT_NODEFILE -n $NPROCS ./miniHACC-AoS 15000
# ls -l $TARGET/ # ls -l $TARGET/
# sleep 5 # sleep 5
#mpirun -f $COBALT_NODEFILE -n $NPROCS ./miniHACC-AoS-R $PARTICLES
rm $TARGET/* rm $TARGET/*
mpirun -f $COBALT_NODEFILE -n $NPROCS ./miniHACC-AoS 100000 mpirun -f $COBALT_NODEFILE -n $NPROCS ./miniHACC-AoS-MPIIO $PARTICLES
sleep 3 sleep 3
#ls -l /projects/visualization/ftessier/debug/ ls -l /projects/visualization/ftessier/debug/
#md5sum /projects/visualization/ftessier/debug/HACC-AOS-00000000-0.dat md5sum /projects/visualization/ftessier/debug/HACC-AOS-00000000-0.dat
# sleep 60 echo
# rm $TARGET/* echo "---------------------------------------------"
# mpirun -f $COBALT_NODEFILE -n $NPROCS ./miniHACC-AoS-MPIIO 60000 echo
# sleep 3 rm $TARGET/*
# ls -l /projects/visualization/ftessier/debug/ mpirun -f $COBALT_NODEFILE -n $NPROCS ./miniHACC-AoS-W $PARTICLES
# md5sum /projects/visualization/ftessier/debug/HACC-AOS-00000000-0.dat sleep 3
ls -l /projects/visualization/ftessier/debug/
md5sum /projects/visualization/ftessier/debug/HACC-AOS-00000000-0.dat
echo
echo "---------------------------------------------"
echo
sleep 3
mpirun -f $COBALT_NODEFILE -n $NPROCS ./miniHACC-AoS-R $PARTICLES
echo
echo "---------------------------------------------"
echo
sleep 3
rm $TARGET/*
mpirun -f $COBALT_NODEFILE -n $NPROCS ./miniHACC-AoS $PARTICLES
ls -l /projects/visualization/ftessier/debug/
md5sum /projects/visualization/ftessier/debug/HACC-AOS-00000000-0.dat
# echo
# echo "---------------------------------------------"
# echo
# sleep 3
# mpirun -f $COBALT_NODEFILE -n $NPROCS ./miniHACC-AoS $PARTICLES
# sleep 1
#mpirun -f $COBALT_NODEFILE -n $NPROCS ./miniHACC-SoA-R 15000 #mpirun -f $COBALT_NODEFILE -n $NPROCS ./miniHACC-SoA-R 15000
# sleep 5 # sleep 5
echo
echo "---------------------------------------------"
echo
#mpirun -f $COBALT_NODEFILE -n $NPROCS ./miniHACC-AoS-R 15000 #mpirun -f $COBALT_NODEFILE -n $NPROCS ./miniHACC-AoS-R 15000
# for run in {0..9} # for run in {0..9}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment