Skip to content

T
TAPIOCA
Commit 60216e2c authored by Francois Tessier's avatar Francois Tessier
Browse files
Options

Update the SoA version to use the last features of TAPIOCA. Update scripts for... 

Update the SoA version to use the last features of TAPIOCA. Update scripts for various platforms. Improved Makefile.
parent a201ad8e
Hide whitespace changes
Inline Side-by-side
Showing with 67 additions and 96 deletions
examples/HACC-IO/Makefile
View file @ 60216e2c
include ../make.inc/Makefile.$(ARCHI)
all: miniHACC-AoS #miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO
CPP_FILES := $(wildcard *.cpp)
TARGETS := $(CPP_FILES:.cpp=)
miniHACC-AoS:miniHACC-AoS.cpp
$(MPICXX) $(MPI_CFLAGS) $? -o $@ $(MPI_LDFLAGS)
miniHACC-SoA:miniHACC-SoA.cpp
$(MPICXX) $(MPI_CFLAGS) $? -o $@ $(MPI_LDFLAGS)
all:$(TARGETS)
### MPI I/O
miniHACC-AoS-MPIIO:miniHACC-AoS-MPIIO.cpp
$(MPICXX) $(MPI_CFLAGS) $? -o $@ $(MPI_LDFLAGS)
miniHACC-SoA-MPIIO:miniHACC-SoA-MPIIO.cpp
%:%.cpp
$(MPICXX) $(MPI_CFLAGS) $? -o $@ $(MPI_LDFLAGS)
install:
mv miniHACC-AoS miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO $(INSTALL_PATH)/bin
mv $(TARGETS) $(INSTALL_PATH)/bin
clean:
rm $(INSTALL_PATH)/bin/miniHACC-AoS
rm $(INSTALL_PATH)/bin/miniHACC-SoA
rm $(INSTALL_PATH)/bin/miniHACC-AoS-MPIIO
rm $(INSTALL_PATH)/bin/miniHACC-SoA-MPIIO
rm $(INSTALL_PATH)/bin/miniHACC-*
\ No newline at end of file
examples/HACC-IO/miniHACC-AoS.cpp
View file @ 60216e2c
......@@ -29,9 +29,8 @@ int main (int argc, char * argv[])
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &world_numtasks);
MPI_Comm_rank(MPI_COMM_WORLD, &world_myrank);
//mycolor = tp.topology.BridgeNodeId ();
mycolor = 42;
mycolor = tp.topology.BridgeNodeId ();
mykey = world_myrank;
MPI_Comm_split (MPI_COMM_WORLD, mycolor, mykey, &sub_comm);
......@@ -43,7 +42,7 @@ int main (int argc, char * argv[])
file_id = 0;
if ( argv[2] != NULL )
file_id = atoi ( argv[2] );
#ifdef BGQ
snprintf (output, 100, "/projects/visualization/ftessier/debug/HACC-SOA-%08d-%d.dat", mycolor, file_id);
#elif XC40
......@@ -90,13 +89,6 @@ int main (int argc, char * argv[])
file_size = particle_size * sub_particles;
tot_size = particle_size * tot_particles;
// if (sub_myrank == 0) {
// MPI_File_open(MPI_COMM_SELF, output,
// MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL, &file_handle);
// MPI_File_set_size(file_handle, file_size);
// MPI_File_close (&file_handle);
// }
MPI_Exscan (&num_particles, &scan_size, 1, MPI_LONG_LONG, MPI_SUM, sub_comm);
if (0 == sub_myrank) {
......@@ -130,7 +122,7 @@ int main (int argc, char * argv[])
}
tp.Init (chunkCount, chunkSize, chunkOffset, 9, hdr, sub_comm);
tp.setAggregationTier (DDR, "");
tp.setAggregationTier (NVR, "/scratch/tmp");
tp.setTargetTier (HDD, file_size, output);
/*****************/
......@@ -164,8 +156,6 @@ int main (int argc, char * argv[])
tp.Write (offset, mask, num_particles, MPI_UNSIGNED_SHORT, &status);
tp.Finalize ();
end_time = MPI_Wtime();
tot_time = end_time - start_time;
MPI_Reduce (&tot_time, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
......@@ -178,6 +168,8 @@ int main (int argc, char * argv[])
MPI_Barrier (MPI_COMM_WORLD);
tp.Finalize ();
/*****************/
/* READ */
/*****************/
......@@ -199,13 +191,13 @@ int main (int argc, char * argv[])
/* INIT TAPIOCA */
/*****************/
tp.Init (chunkCount, chunkSize, chunkOffset, 9, hdr, sub_comm);
tp.setAggregationTier (DDR, "");
tp.setAggregationTier (NVR, "/scratch/tmp");
tp.setTargetTier (HDD, file_size, output);
/*****************/
start_time = MPI_Wtime();
if (0 == sub_myrank)
if (0 == sub_myrank)
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] Read output file\n", mycolor);
offset = scan_size * particle_size;
......@@ -236,8 +228,6 @@ int main (int argc, char * argv[])
tp.Read (offset, mask_r, num_particles, MPI_UNSIGNED_SHORT, &status);
tp.Finalize ();
end_time = MPI_Wtime();
tot_time = end_time - start_time;
MPI_Reduce (&tot_time, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
......@@ -248,6 +238,8 @@ int main (int argc, char * argv[])
io_bw, (double)tot_size/(1024*1024), max_time * 1000);
}
tp.Finalize ();
/*****************/
/* VERIFICATION */
/*****************/
......
examples/HACC-IO/miniHACC-SoA.cpp
View file @ 60216e2c
......@@ -20,7 +20,6 @@ int main (int argc, char * argv[])
double start_time, end_time, tot_time, max_time;
double io_bw;
MPI_Comm sub_comm;
MPI_File file_handle;
MPI_Status status;
char output[100];
Tapioca tp;
......@@ -45,11 +44,11 @@ int main (int argc, char * argv[])
file_id = atoi ( argv[2] );
#ifdef BGQ
snprintf (output, 100, "/projects/visualization/ftessier/debug/HACC-SOA-%08d-%d.dat", mycolor, file_id);
snprintf (output, 100, "/projects/visualization/ftessier/debug/HACC-SOA-%08d-%d.dat", mycolor, file_id);
#elif XC40
snprintf (output, 100, "/lus/theta-fs0/projects/Performance/ftessier/HACC/HACC-AOS-%08d-%d.dat", mycolor, file_id);
snprintf (output, 100, "/lus/theta-fs0/projects/Performance/ftessier/HACC/HACC-AOS-%08d-%d.dat", mycolor, file_id);
#else
snprintf (output, 100, "./HACC-SOA-%08d-%d.dat", mycolor, file_id);
snprintf (output, 100, "./HACC-SOA-%08d-%d.dat", mycolor, file_id);
#endif
......@@ -90,13 +89,6 @@ int main (int argc, char * argv[])
file_size = particle_size * sub_particles;
tot_size = particle_size * tot_particles;
if (sub_myrank == 0) {
MPI_File_open(MPI_COMM_SELF, output,
MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL, &file_handle);
MPI_File_set_size(file_handle, file_size);
MPI_File_close (&file_handle);
}
MPI_Exscan (&num_particles, &scan_size, 1, MPI_LONG_LONG, MPI_SUM, sub_comm);
if (0 == sub_myrank) {
......@@ -131,44 +123,41 @@ int main (int argc, char * argv[])
chunkOffset[i] += scan_size * chunkSize[i];
}
tp.Initialize (chunkCount, chunkSize, chunkOffset, 9, hdr, STRUCTURE_OF_ARRAYS, sub_comm);
tp.Init (chunkCount, chunkSize, chunkOffset, 9, hdr, sub_comm);
tp.setAggregationTier (NVR, "/scratch/tmp");
tp.setTargetTier (HDD, file_size, output);
/*****************/
start_time = MPI_Wtime();
MPI_File_open(sub_comm, output,
MPI_MODE_WRONLY, MPI_INFO_NULL, &file_handle);
offset = scan_size * sizeof(float);
tp.Write (file_handle, offset, xx, num_particles, MPI_FLOAT, &status);
tp.Write (offset, xx, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
tp.Write (file_handle, offset, yy, num_particles, MPI_FLOAT, &status);
tp.Write (offset, yy, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
tp.Write (file_handle, offset, zz, num_particles, MPI_FLOAT, &status);
tp.Write (offset, zz, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
tp.Write (file_handle, offset, vx, num_particles, MPI_FLOAT, &status);
tp.Write (offset, vx, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
tp.Write (file_handle, offset, vy, num_particles, MPI_FLOAT, &status);
tp.Write (offset, vy, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
tp.Write (file_handle, offset, vz, num_particles, MPI_FLOAT, &status);
tp.Write (offset, vz, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
tp.Write (file_handle, offset, phi, num_particles, MPI_FLOAT, &status);
tp.Write (offset, phi, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(int64_t);
tp.Write (file_handle, offset, pid, num_particles, MPI_LONG_LONG, &status);
tp.Write (offset, pid, num_particles, MPI_LONG_LONG, &status);
offset += (sub_particles - scan_size) * sizeof(int64_t) + scan_size * sizeof(uint16_t);
tp.Write (file_handle, offset, mask, num_particles, MPI_UNSIGNED_SHORT, &status);
tp.Write (offset, mask, num_particles, MPI_UNSIGNED_SHORT, &status);
MPI_File_close (&file_handle);
end_time = MPI_Wtime();
tot_time = end_time - start_time;
MPI_Reduce (&tot_time, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
......@@ -203,47 +192,44 @@ int main (int argc, char * argv[])
/*****************/
/* INIT TAPIOCA */
/*****************/
tp.Initialize (chunkCount, chunkSize, chunkOffset, 9, hdr, ARRAY_OF_STRUCTURES, sub_comm);
tp.Init (chunkCount, chunkSize, chunkOffset, 9, hdr, sub_comm);
tp.setAggregationTier (NVR, "/scratch/tmp");
tp.setTargetTier (HDD, file_size, output);
/*****************/
start_time = MPI_Wtime();
MPI_File_open(sub_comm, output,
MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handle);
if (0 == sub_myrank)
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] Read output file\n", mycolor);
offset = scan_size * sizeof(float);
tp.Read (file_handle, offset, xx_r, num_particles, MPI_FLOAT, &status);
tp.Read (offset, xx_r, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
tp.Read (file_handle, offset, yy_r, num_particles, MPI_FLOAT, &status);
tp.Read (offset, yy_r, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
tp.Read (file_handle, offset, zz_r, num_particles, MPI_FLOAT, &status);
tp.Read (offset, zz_r, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
tp.Read (file_handle, offset, vx_r, num_particles, MPI_FLOAT, &status);
tp.Read (offset, vx_r, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
tp.Read (file_handle, offset, vy_r, num_particles, MPI_FLOAT, &status);
tp.Read (offset, vy_r, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
tp.Read (file_handle, offset, vz_r, num_particles, MPI_FLOAT, &status);
tp.Read (offset, vz_r, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
tp.Read (file_handle, offset, phi_r, num_particles, MPI_FLOAT, &status);
tp.Read (offset, phi_r, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(int64_t);
tp.Read (file_handle, offset, pid_r, num_particles, MPI_LONG_LONG, &status);
tp.Read (offset, pid_r, num_particles, MPI_LONG_LONG, &status);
offset += (sub_particles - scan_size) * sizeof(int64_t) + scan_size * sizeof(uint16_t);
tp.Read (file_handle, offset, mask_r, num_particles, MPI_UNSIGNED_SHORT, &status);
tp.Read (offset, mask_r, num_particles, MPI_UNSIGNED_SHORT, &status);
MPI_File_close (&file_handle);
end_time = MPI_Wtime();
tot_time = end_time - start_time;
MPI_Reduce (&tot_time, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
......@@ -254,6 +240,8 @@ int main (int argc, char * argv[])
io_bw, (double)tot_size/(1024*1024), max_time * 1000);
}
tp.Finalize ();
/*****************/
/* VERIFICATION */
/*****************/
......
examples/HACC-IO/run_check_bgq.sh
View file @ 60216e2c
#!/bin/bash
NODES=128
NODES=4096
PPN=16
NPROCS=$((NODES*PPN))
TARGET="/projects/visualization/ftessier/debug"
......@@ -9,8 +9,8 @@ cd $HOME/install/$ARCHI/bin/
export TAPIOCA_DEVNULL=false
export TAPIOCA_COMMSPLIT=true
export TAPIOCA_STRATEGY=UNIFORM
export TAPIOCA_NBAGGR=16
export TAPIOCA_STRATEGY=TOPOLOGY_AWARE
export TAPIOCA_NBAGGR=8
export TAPIOCA_BUFFERSIZE=16777216
export TAPIOCA_PIPELINING=true
......@@ -34,7 +34,7 @@ function updateSettings()
function outputFile ()
{
OUTPUT="$HOME/xp_tapioca/contention"
OUTPUT="$HOME/xp_tapioca/memory_abstraction"
OUTPUT="${OUTPUT}/HACC_BGQ_MIRA_${BIN}_${NODES}_${PPN}"
OUTPUT="${OUTPUT}_${TAPIOCA_STRATEGY}_${TAPIOCA_NBAGGR}_${TAPIOCA_BUFFERSIZE}_${PARTICLES}"
}
......@@ -43,13 +43,13 @@ updateSettings
for run in {0..9}
do
for PARTICLES in 15000 25000 35000 50000
for PARTICLES in 5000 15000 25000 35000 50000 100000
do
for BIN in miniHACC-AoS miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO
for BIN in miniHACC-AoS miniHACC-AoS-MPIIO #miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO
do
rm $TARGET/*
outputFile
runjob --block $COBALT_PARTNAME --envs $SETTINGS -p $PPN --np $NPROCS : ./$BIN $PARTICLES
runjob --block $COBALT_PARTNAME --envs $SETTINGS -p $PPN --np $NPROCS : ./$BIN $PARTICLES >> $OUTPUT
#>> $OUTPUT
sleep 3
done
......
examples/HACC-IO/run_check_haswell.sh
View file @ 60216e2c
......@@ -31,7 +31,9 @@ printenv | egrep "TAPIOCA_"
rm $TARGET/*
mpirun -f $COBALT_NODEFILE -n $NPROCS ./miniHACC-AoS 15000
# ls -l $TARGET/
# sleep 5
sleep 5
rm $TARGET/*
mpirun -f $COBALT_NODEFILE -n $NPROCS ./miniHACC-SoA 15000
# mpirun -f $COBALT_NODEFILE -n $NPROCS ./miniHACC-AoS-R 15000
# sleep 5
echo
......
examples/HACC-IO/run_check_xc40.sh
View file @ 60216e2c
#!/bin/bash
NODES=32
NODES=2048
PPN=16
NPROCS=$((NODES*PPN))
TARGET="/lus/theta-fs0/projects/Performance/ftessier/HACC"
DDT="/soft/debuggers/forge/bin/ddt --connect"
STRIPE_COUNT=4
STRIPE_SIZE=8388608
AGGR_MULT=1
STRIPE_COUNT=48
STRIPE_SIZE=16777216
AGGR_MULT=8
cd $HOME/install/$ARCHI/bin/
export TAPIOCA_DEVNULL=false
export TAPIOCA_COMMSPLIT=true
export TAPIOCA_STRATEGY=TOPOLOGY_AWARE
export TAPIOCA_NBAGGR=4
export TAPIOCA_NBAGGR=384
export TAPIOCA_BUFFERSIZE=16777216
function setLustreFS ()
......@@ -41,18 +41,18 @@ function updateSettings()
function outputFile ()
{
OUTPUT="$HOME/PUB-IPDPS18/dat/mem_abstraction"
OUTPUT="$HOME/xp_tapioca/memory_abstraction"
OUTPUT="${OUTPUT}/HACC_XC40_${BIN}_${NODES}_${PPN}"
OUTPUT="${OUTPUT}_${TAPIOCA_STRATEGY}_${TAPIOCA_NBAGGR}_${TAPIOCA_BUFFERSIZE}_${PARTICLES}"
}
for run in {0..2}
for run in {0..9}
do
for PARTICLES in 25000
for PARTICLES in 5000 15000 25000 35000 50000 100000
do
updateSettings
for BIN in miniHACC-AoS miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO
for BIN in miniHACC-AoS miniHACC-AoS-MPIIO
do
outputFile
setLustreFS >> $OUTPUT
......
examples/HACC-IO/run_debug_xc40.sh
View file @ 60216e2c
#!/bin/bash
NODES=64
NODES=8
PPN=16
NPROCS=$((NODES*PPN))
TARGET="/lus/theta-fs0/projects/Performance/ftessier/HACC"
......@@ -14,7 +14,7 @@ cd $HOME/install/$ARCHI/bin/
export TAPIOCA_DEVNULL=false
export TAPIOCA_COMMSPLIT=true
export TAPIOCA_STRATEGY=TOPOLOGY_AWARE
export TAPIOCA_NBAGGR=8
export TAPIOCA_NBAGGR=2
export TAPIOCA_BUFFERSIZE=16777216
function setLustreFS ()
......@@ -51,4 +51,4 @@ BIN=miniHACC-AoS
updateSettings
setLustreFS
aprun $SETTINGS -n $NPROCS -N $PPN ./$BIN $PARTICLES
$DDT aprun $SETTINGS -n $NPROCS -N $PPN ./$BIN $PARTICLES
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment