Commit 713d0763 authored by Francois Tessier's avatar Francois Tessier

HACC-IO re-written with TAPIOCA for both Write and Read operations

parent bd43b858
MPICXX = CC
MPICXX = mpixlcxx
MPI_CFLAGS = -g -O3 -I$(HOME)/install/include/ -DXC40 -DDEBUG
MPI_LDFLAGS = -L$(HOME)/install/lib/ -ltapioca -llustreapi -Wl,--whole-archive,-ldmapp,--no-whole-archive
MPI_CFLAGS = -g -O3 -I$(HOME)/install_bgq/include/ -DBGQ -DDEBUG
MPI_LDFLAGS = -L$(HOME)/install_bgq/lib/ -ltapioca
all: miniHACC-AoS miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO
......@@ -19,5 +19,5 @@ miniHACC-SoA-MPIIO:miniHACC-SoA-MPIIO.cpp
$(MPICXX) $(MPI_CFLAGS) $? -o $@ $(MPI_LDFLAGS)
clean:
rm miniHACC-AoS miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO *.o HACC.dat
rm miniHACC-AoS miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO
......@@ -39,8 +39,8 @@ int main (int argc, char * argv[])
num_particles = atoi ( argv[1] );
file_id = 0;
if ( argv[1] != NULL )
file_id = atoi ( argv[1] );
if ( argv[2] != NULL )
file_id = atoi ( argv[2] );
#ifdef BGQ
snprintf (output, 100, "/projects/visualization/ftessier/debug/HACC-SOA-%08d-%d.dat", mycolor, file_id);
......
......@@ -24,6 +24,7 @@ int main (int argc, char * argv[])
MPI_Status status;
char output[100];
Tapioca tp;
Tapioca tp2;
int64_t chunkCount[9], chunkOffset[9];
int chunkSize[9];
......@@ -179,6 +180,8 @@ int main (int argc, char * argv[])
MPI_Barrier (MPI_COMM_WORLD);
tp.Finalize ();
/*****************/
/* READ */
/*****************/
......@@ -196,6 +199,12 @@ int main (int argc, char * argv[])
pid_r = new int64_t[num_particles];
mask_r = new uint16_t[num_particles];
/*****************/
/* INIT TAPIOCA */
/*****************/
tp2.ReadInitialize (output, chunkCount, chunkSize, chunkOffset, 9, hdr, ARRAY_OF_STRUCTURES, sub_comm);
/*****************/
start_time = MPI_Wtime();
MPI_File_open(sub_comm, output,
......@@ -206,31 +215,31 @@ int main (int argc, char * argv[])
offset = scan_size * particle_size;
MPI_File_read_at_all (file_handle, offset, xx_r, num_particles, MPI_FLOAT, &status);
tp2.Read (file_handle, offset, xx_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
MPI_File_read_at_all (file_handle, offset, yy_r, num_particles, MPI_FLOAT, &status);
tp2.Read (file_handle, offset, yy_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
MPI_File_read_at_all (file_handle, offset, zz_r, num_particles, MPI_FLOAT, &status);
tp2.Read (file_handle, offset, zz_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
MPI_File_read_at_all (file_handle, offset, vx_r, num_particles, MPI_FLOAT, &status);
tp2.Read (file_handle, offset, vx_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
MPI_File_read_at_all (file_handle, offset, vy_r, num_particles, MPI_FLOAT, &status);
tp2.Read (file_handle, offset, vy_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
MPI_File_read_at_all (file_handle, offset, vz_r, num_particles, MPI_FLOAT, &status);
tp2.Read (file_handle, offset, vz_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
MPI_File_read_at_all (file_handle, offset, phi_r, num_particles, MPI_FLOAT, &status);
tp2.Read (file_handle, offset, phi_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
MPI_File_read_at_all (file_handle, offset, pid_r, num_particles, MPI_LONG_LONG, &status);
tp2.Read (file_handle, offset, pid_r, num_particles, MPI_LONG_LONG, &status);
offset += num_particles * sizeof(int64_t);
MPI_File_read_at_all (file_handle, offset, mask_r, num_particles, MPI_UNSIGNED_SHORT, &status);
tp2.Read (file_handle, offset, mask_r, num_particles, MPI_UNSIGNED_SHORT, &status);
MPI_File_close (&file_handle);
......
......@@ -39,8 +39,8 @@ int main (int argc, char * argv[])
num_particles = atoi ( argv[1] );
file_id = 0;
if ( argv[1] != NULL )
file_id = atoi ( argv[1] );
if ( argv[2] != NULL )
file_id = atoi ( argv[2] );
#ifdef BGQ
snprintf (output, 100, "/projects/visualization/ftessier/debug/HACC-SOA-%08d-%d.dat", mycolor, file_id);
......
......@@ -24,6 +24,7 @@ int main (int argc, char * argv[])
MPI_Status status;
char output[100];
Tapioca tp;
Tapioca tp2;
int64_t chunkCount[9], chunkOffset[9];
int chunkSize[9];
......@@ -41,8 +42,8 @@ int main (int argc, char * argv[])
num_particles = atoi ( argv[1] );
file_id = 0;
if ( argv[1] != NULL )
file_id = atoi ( argv[1] );
if ( argv[2] != NULL )
file_id = atoi ( argv[2] );
#ifdef BGQ
snprintf (output, 100, "/projects/visualization/ftessier/debug/HACC-SOA-%08d-%d.dat", mycolor, file_id);
......@@ -198,6 +199,12 @@ int main (int argc, char * argv[])
pid_r = new int64_t[num_particles];
mask_r = new uint16_t[num_particles];
/*****************/
/* INIT TAPIOCA */
/*****************/
tp2.ReadInitialize (output, chunkCount, chunkSize, chunkOffset, 9, hdr, ARRAY_OF_STRUCTURES, sub_comm);
/*****************/
start_time = MPI_Wtime();
MPI_File_open(sub_comm, output,
......@@ -208,31 +215,31 @@ int main (int argc, char * argv[])
offset = scan_size * sizeof(float);
MPI_File_read_at_all (file_handle, offset, xx_r, num_particles, MPI_FLOAT, &status);
tp2.Read (file_handle, offset, xx_r, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
MPI_File_read_at_all (file_handle, offset, yy_r, num_particles, MPI_FLOAT, &status);
tp2.Read (file_handle, offset, yy_r, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
MPI_File_read_at_all (file_handle, offset, zz_r, num_particles, MPI_FLOAT, &status);
tp2.Read (file_handle, offset, zz_r, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
MPI_File_read_at_all (file_handle, offset, vx_r, num_particles, MPI_FLOAT, &status);
tp2.Read (file_handle, offset, vx_r, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
MPI_File_read_at_all (file_handle, offset, vy_r, num_particles, MPI_FLOAT, &status);
tp2.Read (file_handle, offset, vy_r, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
MPI_File_read_at_all (file_handle, offset, vz_r, num_particles, MPI_FLOAT, &status);
tp2.Read (file_handle, offset, vz_r, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
MPI_File_read_at_all (file_handle, offset, phi_r, num_particles, MPI_FLOAT, &status);
tp2.Read (file_handle, offset, phi_r, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(int64_t);
MPI_File_read_at_all (file_handle, offset, pid_r, num_particles, MPI_LONG_LONG, &status);
tp2.Read (file_handle, offset, pid_r, num_particles, MPI_LONG_LONG, &status);
offset += (sub_particles - scan_size) * sizeof(int64_t) + scan_size * sizeof(uint16_t);
MPI_File_read_at_all (file_handle, offset, mask_r, num_particles, MPI_UNSIGNED_SHORT, &status);
tp2.Read (file_handle, offset, mask_r, num_particles, MPI_UNSIGNED_SHORT, &status);
MPI_File_close (&file_handle);
......
#!/bin/bash
NODES=128
NODES=1024
PPN=16
NPROCS=$((NODES*PPN))
TARGET="/projects/visualization/ftessier/debug"
......@@ -31,10 +31,24 @@ function updateSettings()
SETTINGS="$SETTINGS BGLOCKLESSMPIO_F_TYPE=0x47504653"
}
rm $TARGET/*
updateSettings
runjob --block $COBALT_PARTNAME --envs $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-AoS 25000 1
for run in 0 1 2
do
rm $TARGET/*
updateSettings
runjob --block $COBALT_PARTNAME --envs $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-AoS 25000
rm $TARGET/*
updateSettings
runjob --block $COBALT_PARTNAME --envs $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-AoS-MPIIO 25000
rm $TARGET/*
updateSettings
runjob --block $COBALT_PARTNAME --envs $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-SoA 25000
rm $TARGET/*
updateSettings
runjob --block $COBALT_PARTNAME --envs $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-SoA-MPIIO 25000
done
# #########################
# # Array of Structures
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment