Commit 713d0763 authored by Francois Tessier's avatar Francois Tessier

HACC-IO re-written with TAPIOCA for both Write and Read operations

parent bd43b858
MPICXX = CC MPICXX = mpixlcxx
MPI_CFLAGS = -g -O3 -I$(HOME)/install/include/ -DXC40 -DDEBUG MPI_CFLAGS = -g -O3 -I$(HOME)/install_bgq/include/ -DBGQ -DDEBUG
MPI_LDFLAGS = -L$(HOME)/install/lib/ -ltapioca -llustreapi -Wl,--whole-archive,-ldmapp,--no-whole-archive MPI_LDFLAGS = -L$(HOME)/install_bgq/lib/ -ltapioca
all: miniHACC-AoS miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO all: miniHACC-AoS miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO
...@@ -19,5 +19,5 @@ miniHACC-SoA-MPIIO:miniHACC-SoA-MPIIO.cpp ...@@ -19,5 +19,5 @@ miniHACC-SoA-MPIIO:miniHACC-SoA-MPIIO.cpp
$(MPICXX) $(MPI_CFLAGS) $? -o $@ $(MPI_LDFLAGS) $(MPICXX) $(MPI_CFLAGS) $? -o $@ $(MPI_LDFLAGS)
clean: clean:
rm miniHACC-AoS miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO *.o HACC.dat rm miniHACC-AoS miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO
...@@ -39,8 +39,8 @@ int main (int argc, char * argv[]) ...@@ -39,8 +39,8 @@ int main (int argc, char * argv[])
num_particles = atoi ( argv[1] ); num_particles = atoi ( argv[1] );
file_id = 0; file_id = 0;
if ( argv[1] != NULL ) if ( argv[2] != NULL )
file_id = atoi ( argv[1] ); file_id = atoi ( argv[2] );
#ifdef BGQ #ifdef BGQ
snprintf (output, 100, "/projects/visualization/ftessier/debug/HACC-SOA-%08d-%d.dat", mycolor, file_id); snprintf (output, 100, "/projects/visualization/ftessier/debug/HACC-SOA-%08d-%d.dat", mycolor, file_id);
......
...@@ -24,6 +24,7 @@ int main (int argc, char * argv[]) ...@@ -24,6 +24,7 @@ int main (int argc, char * argv[])
MPI_Status status; MPI_Status status;
char output[100]; char output[100];
Tapioca tp; Tapioca tp;
Tapioca tp2;
int64_t chunkCount[9], chunkOffset[9]; int64_t chunkCount[9], chunkOffset[9];
int chunkSize[9]; int chunkSize[9];
...@@ -179,6 +180,8 @@ int main (int argc, char * argv[]) ...@@ -179,6 +180,8 @@ int main (int argc, char * argv[])
MPI_Barrier (MPI_COMM_WORLD); MPI_Barrier (MPI_COMM_WORLD);
tp.Finalize ();
/*****************/ /*****************/
/* READ */ /* READ */
/*****************/ /*****************/
...@@ -196,6 +199,12 @@ int main (int argc, char * argv[]) ...@@ -196,6 +199,12 @@ int main (int argc, char * argv[])
pid_r = new int64_t[num_particles]; pid_r = new int64_t[num_particles];
mask_r = new uint16_t[num_particles]; mask_r = new uint16_t[num_particles];
/*****************/
/* INIT TAPIOCA */
/*****************/
tp2.ReadInitialize (output, chunkCount, chunkSize, chunkOffset, 9, hdr, ARRAY_OF_STRUCTURES, sub_comm);
/*****************/
start_time = MPI_Wtime(); start_time = MPI_Wtime();
MPI_File_open(sub_comm, output, MPI_File_open(sub_comm, output,
...@@ -206,31 +215,31 @@ int main (int argc, char * argv[]) ...@@ -206,31 +215,31 @@ int main (int argc, char * argv[])
offset = scan_size * particle_size; offset = scan_size * particle_size;
MPI_File_read_at_all (file_handle, offset, xx_r, num_particles, MPI_FLOAT, &status); tp2.Read (file_handle, offset, xx_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float); offset += num_particles * sizeof(float);
MPI_File_read_at_all (file_handle, offset, yy_r, num_particles, MPI_FLOAT, &status); tp2.Read (file_handle, offset, yy_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float); offset += num_particles * sizeof(float);
MPI_File_read_at_all (file_handle, offset, zz_r, num_particles, MPI_FLOAT, &status); tp2.Read (file_handle, offset, zz_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float); offset += num_particles * sizeof(float);
MPI_File_read_at_all (file_handle, offset, vx_r, num_particles, MPI_FLOAT, &status); tp2.Read (file_handle, offset, vx_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float); offset += num_particles * sizeof(float);
MPI_File_read_at_all (file_handle, offset, vy_r, num_particles, MPI_FLOAT, &status); tp2.Read (file_handle, offset, vy_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float); offset += num_particles * sizeof(float);
MPI_File_read_at_all (file_handle, offset, vz_r, num_particles, MPI_FLOAT, &status); tp2.Read (file_handle, offset, vz_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float); offset += num_particles * sizeof(float);
MPI_File_read_at_all (file_handle, offset, phi_r, num_particles, MPI_FLOAT, &status); tp2.Read (file_handle, offset, phi_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float); offset += num_particles * sizeof(float);
MPI_File_read_at_all (file_handle, offset, pid_r, num_particles, MPI_LONG_LONG, &status); tp2.Read (file_handle, offset, pid_r, num_particles, MPI_LONG_LONG, &status);
offset += num_particles * sizeof(int64_t); offset += num_particles * sizeof(int64_t);
MPI_File_read_at_all (file_handle, offset, mask_r, num_particles, MPI_UNSIGNED_SHORT, &status); tp2.Read (file_handle, offset, mask_r, num_particles, MPI_UNSIGNED_SHORT, &status);
MPI_File_close (&file_handle); MPI_File_close (&file_handle);
......
...@@ -39,8 +39,8 @@ int main (int argc, char * argv[]) ...@@ -39,8 +39,8 @@ int main (int argc, char * argv[])
num_particles = atoi ( argv[1] ); num_particles = atoi ( argv[1] );
file_id = 0; file_id = 0;
if ( argv[1] != NULL ) if ( argv[2] != NULL )
file_id = atoi ( argv[1] ); file_id = atoi ( argv[2] );
#ifdef BGQ #ifdef BGQ
snprintf (output, 100, "/projects/visualization/ftessier/debug/HACC-SOA-%08d-%d.dat", mycolor, file_id); snprintf (output, 100, "/projects/visualization/ftessier/debug/HACC-SOA-%08d-%d.dat", mycolor, file_id);
......
...@@ -24,6 +24,7 @@ int main (int argc, char * argv[]) ...@@ -24,6 +24,7 @@ int main (int argc, char * argv[])
MPI_Status status; MPI_Status status;
char output[100]; char output[100];
Tapioca tp; Tapioca tp;
Tapioca tp2;
int64_t chunkCount[9], chunkOffset[9]; int64_t chunkCount[9], chunkOffset[9];
int chunkSize[9]; int chunkSize[9];
...@@ -41,8 +42,8 @@ int main (int argc, char * argv[]) ...@@ -41,8 +42,8 @@ int main (int argc, char * argv[])
num_particles = atoi ( argv[1] ); num_particles = atoi ( argv[1] );
file_id = 0; file_id = 0;
if ( argv[1] != NULL ) if ( argv[2] != NULL )
file_id = atoi ( argv[1] ); file_id = atoi ( argv[2] );
#ifdef BGQ #ifdef BGQ
snprintf (output, 100, "/projects/visualization/ftessier/debug/HACC-SOA-%08d-%d.dat", mycolor, file_id); snprintf (output, 100, "/projects/visualization/ftessier/debug/HACC-SOA-%08d-%d.dat", mycolor, file_id);
...@@ -198,6 +199,12 @@ int main (int argc, char * argv[]) ...@@ -198,6 +199,12 @@ int main (int argc, char * argv[])
pid_r = new int64_t[num_particles]; pid_r = new int64_t[num_particles];
mask_r = new uint16_t[num_particles]; mask_r = new uint16_t[num_particles];
/*****************/
/* INIT TAPIOCA */
/*****************/
tp2.ReadInitialize (output, chunkCount, chunkSize, chunkOffset, 9, hdr, ARRAY_OF_STRUCTURES, sub_comm);
/*****************/
start_time = MPI_Wtime(); start_time = MPI_Wtime();
MPI_File_open(sub_comm, output, MPI_File_open(sub_comm, output,
...@@ -208,31 +215,31 @@ int main (int argc, char * argv[]) ...@@ -208,31 +215,31 @@ int main (int argc, char * argv[])
offset = scan_size * sizeof(float); offset = scan_size * sizeof(float);
MPI_File_read_at_all (file_handle, offset, xx_r, num_particles, MPI_FLOAT, &status); tp2.Read (file_handle, offset, xx_r, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float); offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
MPI_File_read_at_all (file_handle, offset, yy_r, num_particles, MPI_FLOAT, &status); tp2.Read (file_handle, offset, yy_r, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float); offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
MPI_File_read_at_all (file_handle, offset, zz_r, num_particles, MPI_FLOAT, &status); tp2.Read (file_handle, offset, zz_r, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float); offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
MPI_File_read_at_all (file_handle, offset, vx_r, num_particles, MPI_FLOAT, &status); tp2.Read (file_handle, offset, vx_r, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float); offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
MPI_File_read_at_all (file_handle, offset, vy_r, num_particles, MPI_FLOAT, &status); tp2.Read (file_handle, offset, vy_r, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float); offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
MPI_File_read_at_all (file_handle, offset, vz_r, num_particles, MPI_FLOAT, &status); tp2.Read (file_handle, offset, vz_r, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float); offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
MPI_File_read_at_all (file_handle, offset, phi_r, num_particles, MPI_FLOAT, &status); tp2.Read (file_handle, offset, phi_r, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(int64_t); offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(int64_t);
MPI_File_read_at_all (file_handle, offset, pid_r, num_particles, MPI_LONG_LONG, &status); tp2.Read (file_handle, offset, pid_r, num_particles, MPI_LONG_LONG, &status);
offset += (sub_particles - scan_size) * sizeof(int64_t) + scan_size * sizeof(uint16_t); offset += (sub_particles - scan_size) * sizeof(int64_t) + scan_size * sizeof(uint16_t);
MPI_File_read_at_all (file_handle, offset, mask_r, num_particles, MPI_UNSIGNED_SHORT, &status); tp2.Read (file_handle, offset, mask_r, num_particles, MPI_UNSIGNED_SHORT, &status);
MPI_File_close (&file_handle); MPI_File_close (&file_handle);
......
#!/bin/bash #!/bin/bash
NODES=128 NODES=1024
PPN=16 PPN=16
NPROCS=$((NODES*PPN)) NPROCS=$((NODES*PPN))
TARGET="/projects/visualization/ftessier/debug" TARGET="/projects/visualization/ftessier/debug"
...@@ -31,10 +31,24 @@ function updateSettings() ...@@ -31,10 +31,24 @@ function updateSettings()
SETTINGS="$SETTINGS BGLOCKLESSMPIO_F_TYPE=0x47504653" SETTINGS="$SETTINGS BGLOCKLESSMPIO_F_TYPE=0x47504653"
} }
rm $TARGET/* for run in 0 1 2
updateSettings do
runjob --block $COBALT_PARTNAME --envs $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-AoS 25000 1 rm $TARGET/*
updateSettings
runjob --block $COBALT_PARTNAME --envs $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-AoS 25000
rm $TARGET/*
updateSettings
runjob --block $COBALT_PARTNAME --envs $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-AoS-MPIIO 25000
rm $TARGET/*
updateSettings
runjob --block $COBALT_PARTNAME --envs $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-SoA 25000
rm $TARGET/*
updateSettings
runjob --block $COBALT_PARTNAME --envs $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-SoA-MPIIO 25000
done
# ######################### # #########################
# # Array of Structures # # Array of Structures
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment