Skip to content

GitLab

Commit c17b9640 authored by Francois Tessier's avatar Francois Tessier
Browse files

Add miniHACC implemented with MPI I/O. Add a script to run all the binaries... 

Add miniHACC implemented with MPI I/O. Add a script to run all the binaries and compare output files with md5 sum.
parent 0b1d6875
Show whitespace changes
Inline Side-by-side
Showing with 577 additions and 0 deletions
examples/miniHACC-AoS-MPIIO.cpp 0 → 100644
View file @ c17b9640
#include <stdio.h>
#include <stdlib.h>
#include <assert.h>
#include <stdint.h>
#include <mpi.h>
#include <mpix.h>
#define RED "\x1b[31m"
#define GREEN "\x1b[32m"
#define BLUE "\x1b[34m"
#define RESET "\x1b[0m"
int main (int argc, char * argv[])
{
int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank;
int64_t num_particles = 25000;
int64_t sub_particles, tot_particles, particle_size, file_size, tot_size;
int64_t scan_size = 0, offset;
double start_time, end_time, tot_time, max_time;
double io_bw;
MPI_Comm sub_comm;
MPI_File file_handle;
MPI_Status status;
char output[100];
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &world_numtasks);
MPI_Comm_rank(MPI_COMM_WORLD, &world_myrank);
mycolor = MPIX_IO_link_id ();
mykey = world_myrank;
MPI_Comm_split (MPI_COMM_WORLD, mycolor, mykey, &sub_comm);
MPI_Comm_size(sub_comm, &sub_numtasks);
MPI_Comm_rank(sub_comm, &sub_myrank);
snprintf (output, 100, "/projects/visualization/ftessier/debug/HACC-AOS-%08d.dat", mycolor);
/*****************/
/* WRITE */
/*****************/
float *xx, *yy, *zz, *vx, *vy, *vz, *phi;
int64_t* pid;
uint16_t* mask;
xx = new float[num_particles];
yy = new float[num_particles];
zz = new float[num_particles];
vx = new float[num_particles];
vy = new float[num_particles];
vz = new float[num_particles];
phi = new float[num_particles];
pid = new int64_t[num_particles];
mask = new uint16_t[num_particles];
for (uint64_t i = 0; i< num_particles; i++)
{
xx[i] = (float)i;
yy[i] = (float)i;
zz[i] = (float)i;
vx[i] = (float)i;
vy[i] = (float)i;
vz[i] = (float)i;
phi[i] = (float)i;
pid[i] = (int64_t)i;
mask[i] = (uint16_t)world_myrank;
}
MPI_Allreduce(&num_particles, &sub_particles, 1, MPI_LONG_LONG, MPI_SUM, sub_comm);
MPI_Allreduce(&num_particles, &tot_particles, 1, MPI_LONG_LONG, MPI_SUM, MPI_COMM_WORLD);
particle_size = (7 * sizeof(float)) + sizeof(int64_t) + sizeof(uint16_t);
file_size = particle_size * sub_particles;
tot_size = particle_size * tot_particles;
if (sub_myrank == 0) {
MPI_File_open(MPI_COMM_SELF, output,
MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL, &file_handle);
MPI_File_set_size(file_handle, file_size);
MPI_File_close (&file_handle);
}
MPI_Exscan (&num_particles, &scan_size, 1, MPI_LONG_LONG, MPI_SUM, sub_comm);
if (0 == sub_myrank) {
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] miniHACC-AoS-MPIIO\n", mycolor);
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] Write output file (AoS data layout)\n", mycolor);
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] --> %lld particles per rank\n", mycolor, num_particles);
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] --> File size: %.2f MB (%lld particles)\n",
mycolor, (double)file_size/(1024*1024), sub_particles);
}
start_time = MPI_Wtime();
MPI_File_open(sub_comm, output,
MPI_MODE_WRONLY, MPI_INFO_NULL, &file_handle);
offset = scan_size * particle_size;
MPI_File_write_at_all (file_handle, offset, xx, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
MPI_File_write_at_all (file_handle, offset, yy, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
MPI_File_write_at_all (file_handle, offset, zz, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
MPI_File_write_at_all (file_handle, offset, vx, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
MPI_File_write_at_all (file_handle, offset, vy, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
MPI_File_write_at_all (file_handle, offset, vz, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
MPI_File_write_at_all (file_handle, offset, phi, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
MPI_File_write_at_all (file_handle, offset, pid, num_particles, MPI_LONG_LONG, &status);
offset += num_particles * sizeof(int64_t);
MPI_File_write_at_all (file_handle, offset, mask, num_particles, MPI_UNSIGNED_SHORT, &status);
MPI_File_close (&file_handle);
end_time = MPI_Wtime();
tot_time = end_time - start_time;
MPI_Reduce (&tot_time, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
if (0 == world_myrank) {
io_bw = (double)tot_size / max_time / (1024 * 1024);
fprintf (stdout, BLUE "[TIMING]" RESET " Write I/O bandwidth: %.2f MBps (%.2f MB in %.2f ms)\n",
io_bw, (double)tot_size/(1024*1024), max_time * 1000);
}
MPI_Barrier (MPI_COMM_WORLD);
/*****************/
/* READ */
/*****************/
float *xx_r, *yy_r, *zz_r, *vx_r, *vy_r, *vz_r, *phi_r;
int64_t* pid_r;
uint16_t* mask_r;
xx_r = new float[num_particles];
yy_r = new float[num_particles];
zz_r = new float[num_particles];
vx_r = new float[num_particles];
vy_r = new float[num_particles];
vz_r = new float[num_particles];
phi_r = new float[num_particles];
pid_r = new int64_t[num_particles];
mask_r = new uint16_t[num_particles];
start_time = MPI_Wtime();
MPI_File_open(sub_comm, output,
MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handle);
if (0 == sub_myrank)
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] Read output file\n", mycolor);
offset = scan_size * particle_size;
MPI_File_read_at_all (file_handle, offset, xx_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
MPI_File_read_at_all (file_handle, offset, yy_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
MPI_File_read_at_all (file_handle, offset, zz_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
MPI_File_read_at_all (file_handle, offset, vx_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
MPI_File_read_at_all (file_handle, offset, vy_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
MPI_File_read_at_all (file_handle, offset, vz_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
MPI_File_read_at_all (file_handle, offset, phi_r, num_particles, MPI_FLOAT, &status);
offset += num_particles * sizeof(float);
MPI_File_read_at_all (file_handle, offset, pid_r, num_particles, MPI_LONG_LONG, &status);
offset += num_particles * sizeof(int64_t);
MPI_File_read_at_all (file_handle, offset, mask_r, num_particles, MPI_UNSIGNED_SHORT, &status);
MPI_File_close (&file_handle);
end_time = MPI_Wtime();
tot_time = end_time - start_time;
MPI_Reduce (&tot_time, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
if (0 == world_myrank) {
io_bw = (double)tot_size / max_time / (1024 * 1024);
fprintf (stdout, BLUE "[TIMING]" RESET " Read I/O bandwidth: %.2f MBps (%.2f MB in %.2f ms)\n",
io_bw, (double)tot_size/(1024*1024), max_time * 1000);
}
/*****************/
/* VERIFICATION */
/*****************/
for (uint64_t i = 0; i< num_particles; i++) {
if ((xx[i] != xx_r[i]) || (yy[i] != yy_r[i]) || (zz[i] != zz_r[i])
|| (vx[i] != vx_r[i]) || (vy[i] != vy_r[i]) || (vz[i] != vz_r[i])
|| (phi[i] != phi_r[i])|| (pid[i] != pid_r[i]) || (mask[i] != mask_r[i]))
{
fprintf (stdout, RED "[ERROR]" RESET " Wrong value for particle %d\n", i);
MPI_Abort (MPI_COMM_WORLD, -1);
}
}
if (0 == sub_myrank)
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] Content verified and consistent\n", mycolor);
/*****************/
/* FREE */
/*****************/
delete [] xx;
delete [] xx_r;
delete [] yy;
delete [] yy_r;
delete [] zz;
delete [] zz_r;
delete [] vx;
delete [] vx_r;
delete [] vy;
delete [] vy_r;
delete [] vz;
delete [] vz_r;
delete [] phi;
delete [] phi_r;
delete [] pid;
delete [] pid_r;
delete [] mask;
delete [] mask_r;
MPI_Finalize ();
}
examples/miniHACC-SoA-MPIIO.cpp 0 → 100644
View file @ c17b9640
#include <stdio.h>
#include <stdlib.h>
#include <assert.h>
#include <stdint.h>
#include <mpi.h>
#include <mpix.h>
#define RED "\x1b[31m"
#define GREEN "\x1b[32m"
#define BLUE "\x1b[34m"
#define RESET "\x1b[0m"
int main (int argc, char * argv[])
{
int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank;
int64_t num_particles = 25000;
int64_t sub_particles, tot_particles, particle_size, file_size, tot_size;
int64_t scan_size = 0, offset;
double start_time, end_time, tot_time, max_time;
double io_bw;
MPI_Comm sub_comm;
MPI_File file_handle;
MPI_Status status;
char output[100];
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &world_numtasks);
MPI_Comm_rank(MPI_COMM_WORLD, &world_myrank);
mycolor = MPIX_IO_link_id ();
mykey = world_myrank;
MPI_Comm_split (MPI_COMM_WORLD, mycolor, mykey, &sub_comm);
MPI_Comm_size(sub_comm, &sub_numtasks);
MPI_Comm_rank(sub_comm, &sub_myrank);
snprintf (output, 100, "/projects/visualization/ftessier/debug/HACC-AOS-%08d.dat", mycolor);
/*****************/
/* WRITE */
/*****************/
float *xx, *yy, *zz, *vx, *vy, *vz, *phi;
int64_t* pid;
uint16_t* mask;
xx = new float[num_particles];
yy = new float[num_particles];
zz = new float[num_particles];
vx = new float[num_particles];
vy = new float[num_particles];
vz = new float[num_particles];
phi = new float[num_particles];
pid = new int64_t[num_particles];
mask = new uint16_t[num_particles];
for (uint64_t i = 0; i< num_particles; i++)
{
xx[i] = (float)i;
yy[i] = (float)i;
zz[i] = (float)i;
vx[i] = (float)i;
vy[i] = (float)i;
vz[i] = (float)i;
phi[i] = (float)i;
pid[i] = (int64_t)i;
mask[i] = (uint16_t)world_myrank;
}
MPI_Allreduce(&num_particles, &sub_particles, 1, MPI_LONG_LONG, MPI_SUM, sub_comm);
MPI_Allreduce(&num_particles, &tot_particles, 1, MPI_LONG_LONG, MPI_SUM, MPI_COMM_WORLD);
particle_size = (7 * sizeof(float)) + sizeof(int64_t) + sizeof(uint16_t);
file_size = particle_size * sub_particles;
tot_size = particle_size * tot_particles;
if (sub_myrank == 0) {
MPI_File_open(MPI_COMM_SELF, output,
MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL, &file_handle);
MPI_File_set_size(file_handle, file_size);
MPI_File_close (&file_handle);
}
MPI_Exscan (&num_particles, &scan_size, 1, MPI_LONG_LONG, MPI_SUM, sub_comm);
if (0 == sub_myrank) {
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] miniHACC-SoA-MPIIO\n", mycolor);
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] Write output file (SoA data layout)\n", mycolor);
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] --> %lld particles per rank\n", mycolor, num_particles);
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] --> File size: %.2f MB (%lld particles)\n",
mycolor, (double)file_size/(1024*1024), sub_particles);
}
start_time = MPI_Wtime();
MPI_File_open(sub_comm, output,
MPI_MODE_WRONLY, MPI_INFO_NULL, &file_handle);
offset = scan_size * sizeof(float);
MPI_File_write_at_all (file_handle, offset, xx, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
MPI_File_write_at_all (file_handle, offset, yy, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
MPI_File_write_at_all (file_handle, offset, zz, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
MPI_File_write_at_all (file_handle, offset, vx, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
MPI_File_write_at_all (file_handle, offset, vy, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
MPI_File_write_at_all (file_handle, offset, vz, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
MPI_File_write_at_all (file_handle, offset, phi, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(int64_t);
MPI_File_write_at_all (file_handle, offset, pid, num_particles, MPI_LONG_LONG, &status);
offset += (sub_particles - scan_size) * sizeof(int64_t) + scan_size * sizeof(uint16_t);
MPI_File_write_at_all (file_handle, offset, mask, num_particles, MPI_UNSIGNED_SHORT, &status);
MPI_File_close (&file_handle);
end_time = MPI_Wtime();
tot_time = end_time - start_time;
MPI_Reduce (&tot_time, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
if (0 == world_myrank) {
io_bw = (double)tot_size / max_time / (1024 * 1024);
fprintf (stdout, BLUE "[TIMING]" RESET " Write I/O bandwidth: %.2f MBps (%.2f MB in %.2f ms)\n",
io_bw, (double)tot_size/(1024*1024), max_time * 1000);
}
MPI_Barrier (MPI_COMM_WORLD);
/*****************/
/* READ */
/*****************/
float *xx_r, *yy_r, *zz_r, *vx_r, *vy_r, *vz_r, *phi_r;
int64_t* pid_r;
uint16_t* mask_r;
xx_r = new float[num_particles];
yy_r = new float[num_particles];
zz_r = new float[num_particles];
vx_r = new float[num_particles];
vy_r = new float[num_particles];
vz_r = new float[num_particles];
phi_r = new float[num_particles];
pid_r = new int64_t[num_particles];
mask_r = new uint16_t[num_particles];
start_time = MPI_Wtime();
MPI_File_open(sub_comm, output,
MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handle);
if (0 == sub_myrank)
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] Read output file\n", mycolor);
offset = scan_size * sizeof(float);
MPI_File_read_at_all (file_handle, offset, xx_r, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
MPI_File_read_at_all (file_handle, offset, yy_r, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
MPI_File_read_at_all (file_handle, offset, zz_r, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
MPI_File_read_at_all (file_handle, offset, vx_r, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
MPI_File_read_at_all (file_handle, offset, vy_r, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
MPI_File_read_at_all (file_handle, offset, vz_r, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(float);
MPI_File_read_at_all (file_handle, offset, phi_r, num_particles, MPI_FLOAT, &status);
offset += (sub_particles - scan_size) * sizeof(float) + scan_size * sizeof(int64_t);
MPI_File_read_at_all (file_handle, offset, pid_r, num_particles, MPI_LONG_LONG, &status);
offset += (sub_particles - scan_size) * sizeof(int64_t) + scan_size * sizeof(uint16_t);
MPI_File_read_at_all (file_handle, offset, mask_r, num_particles, MPI_UNSIGNED_SHORT, &status);
MPI_File_close (&file_handle);
end_time = MPI_Wtime();
tot_time = end_time - start_time;
MPI_Reduce (&tot_time, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
if (0 == world_myrank) {
io_bw = (double)tot_size / max_time / (1024 * 1024);
fprintf (stdout, BLUE "[TIMING]" RESET " Read I/O bandwidth: %.2f MBps (%.2f MB in %.2f ms)\n",
io_bw, (double)tot_size/(1024*1024), max_time * 1000);
}
/*****************/
/* VERIFICATION */
/*****************/
for (uint64_t i = 0; i< num_particles; i++) {
if ((xx[i] != xx_r[i]) || (yy[i] != yy_r[i]) || (zz[i] != zz_r[i])
|| (vx[i] != vx_r[i]) || (vy[i] != vy_r[i]) || (vz[i] != vz_r[i])
|| (phi[i] != phi_r[i])|| (pid[i] != pid_r[i]) || (mask[i] != mask_r[i]))
{
fprintf (stdout, RED "[ERROR]" RESET " Wrong value for particle %d\n", i);
MPI_Abort (MPI_COMM_WORLD, -1);
}
}
if (0 == sub_myrank)
fprintf (stdout, GREEN "[INFO]" RESET " [%08d] Content verified and consistent\n", mycolor);
/*****************/
/* FREE */
/*****************/
delete [] xx;
delete [] xx_r;
delete [] yy;
delete [] yy_r;
delete [] zz;
delete [] zz_r;
delete [] vx;
delete [] vx_r;
delete [] vy;
delete [] vy_r;
delete [] vz;
delete [] vz_r;
delete [] phi;
delete [] phi_r;
delete [] pid;
delete [] pid_r;
delete [] mask;
delete [] mask_r;
MPI_Finalize ();
}
examples/run_check.sh 0 → 100755
View file @ c17b9640
#!/bin/bash
VARS="PAMID_VERBOSE=1 BG_SHAREDMEMSIZE=64 PAMID_COLLECTIVES=1 BGLOCKLESSMPIO_F_TYPE=0x47504653 PAMID_COLLECTIVES_MEMORY_OPTIMIZED=1"
NODES=256
PPN=16
NPROCS=$((NODES*PPN))
TARGET="/projects/visualization/ftessier/debug"
cd $HOME/TAPIOCA/examples
export TAPIOCA_DEVNULL=false
export TAPIOCA_COMMSPLIT=true
export TAPIOCA_STRATEGY=TOPOLOGY_AWARE
export TAPIOCA_NBAGGR=8
export TAPIOCA_BUFFERSIZE=16777216
function updateSettings()
{
printenv | egrep "TAPIOCA_"
SETTINGS="$SETTINGS TAPIOCA_DEVNULL=$TAPIOCA_DEVNULL"
SETTINGS="$SETTINGS TAPIOCA_COMMSPLIT=$TAPIOCA_COMMSPLIT"
SETTINGS="$SETTINGS TAPIOCA_STRATEGY=$TAPIOCA_STRATEGY"
SETTINGS="$SETTINGS TAPIOCA_NBAGGR=$TAPIOCA_NBAGGR"
SETTINGS="$SETTINGS TAPIOCA_BUFFERSIZE=$TAPIOCA_BUFFERSIZE"
}
#########################
# Array of Structures
#########################
rm $TARGET/*
runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-AoS-MPIIO
sleep 4
let j=0
for i in $TARGET/*
do
CONTROLAOS[$j]=`md5sum $i | cut -d ' ' -f1`
echo ${CONTROLAOS[$j]}
let j=$j+1
done
rm $TARGET/*
runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-AoS
sleep 4
let j=0
for i in $TARGET/*
do
HASH[$j]=`md5sum $i | cut -d ' ' -f1`
if [ ${CONTROLAOS[$j]} == ${HASH[$j]} ]
then
echo -e "\e[32m[PASSED]\e[39m ${HASH[$j]}"
else
echo -e "\e[31m[FAILED]\e[39m ${HASH[$j]}"
fi
let j=$j+1
done
#########################
# Structure of Arrays
#########################
rm $TARGET/*
runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-SoA-MPIIO
sleep 4
let j=0
for i in $TARGET/*
do
CONTROLSOA[$j]=`md5sum $i | cut -d ' ' -f1`
echo ${CONTROLSOA[$j]}
let j=$j+1
done
rm $TARGET/*
runjob --block $COBALT_PARTNAME --envs $VARS $SETTINGS -p $PPN --np $NPROCS : ./miniHACC-SoA
sleep 4
let j=0
for i in $TARGET/*
do
HASH[$j]=`md5sum $i | cut -d ' ' -f1`
if [ ${CONTROLSOA[$j]} == ${HASH[$j]} ]
then
echo -e "\e[32m[PASSED]\e[39m ${HASH[$j]}"
else
echo -e "\e[31m[FAILED]\e[39m ${HASH[$j]}"
fi
let j=$j+1
done
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment